gmsc.qmd 6.60 KiB
```{julia}
include("$(pwd())/notebooks/setup.jl")
eval(setup_notebooks)
```
# GMSC
```{julia}
# Hyper:
_retrain = true
# Data:
test_size = 0.2
counterfactual_data, test_data = train_test_split(load_gmsc(nothing); test_size=test_size)
X, y = CounterfactualExplanations.DataPreprocessing.unpack_data(counterfactual_data)
X = table(permutedims(X))
labels = counterfactual_data.output_encoder.labels
input_dim, n_obs = size(counterfactual_data.X)
output_dim = length(unique(labels))
```
First, let's create a couple of image classifier architectures:
```{julia}
# Model parameters:
epochs = 100
batch_size = minimum([Int(round(n_obs/10)), 250])
n_hidden = 128
activation = Flux.swish
builder = MLJFlux.@builder Flux.Chain(
Dense(n_in, n_hidden, activation),
Dense(n_hidden, n_hidden, activation),
Dense(n_hidden, n_out),
)
n_ens = 5 # number of models in ensemble
_loss = Flux.Losses.crossentropy # loss function
_finaliser = Flux.softmax # finaliser function
```
```{julia}
# JEM parameters:
𝒟x = Normal()
𝒟y = Categorical(ones(output_dim) ./ output_dim)
sampler = ConditionalSampler(
𝒟x, 𝒟y,
input_size=(input_dim,),
batch_size=10,
)
α = [1.0,1.0,1e-1] # penalty strengths
```
```{julia}
# Simple MLP:
mlp = NeuralNetworkClassifier(
builder=builder,
epochs=epochs,
batch_size=batch_size,
finaliser=_finaliser,
loss=_loss,
)
# Deep Ensemble:
mlp_ens = EnsembleModel(model=mlp, n=n_ens)
# Joint Energy Model:
jem = JointEnergyClassifier(
sampler;
builder=builder,
epochs=epochs,
batch_size=batch_size, finaliser=_finaliser,
loss=_loss,
jem_training_params=(
α=α,verbosity=10,
),
sampling_steps=30,
)
# JEM with adversarial training:
jem_adv = deepcopy(jem)
# jem_adv.adv_training = true
# Deep Ensemble of Joint Energy Models:
jem_ens = EnsembleModel(model=jem, n=n_ens)
# Deep Ensemble of Joint Energy Models with adversarial training:
# jem_ens_plus = EnsembleModel(model=jem_adv, n=n_ens)
# Dictionary of models:
models = Dict(
"MLP" => mlp,
"MLP Ensemble" => mlp_ens,
"JEM" => jem,
"JEM Ensemble" => jem_ens,
# "JEM Ensemble+" => jem_ens_plus,
)
```
```{julia}
# Train models:
function _train(model, X=X, y=labels; cov=.95, method=:simple_inductive, mod_name="model")
conf_model = conformal_model(model; method=method, coverage=cov)
mach = machine(conf_model, X, y)
@info "Begin training $mod_name."
fit!(mach)
@info "Finished training $mod_name."
M = ECCCo.ConformalModel(mach.model, mach.fitresult)
return M
end
if _retrain
model_dict = Dict(mod_name => _train(mod; mod_name=mod_name) for (mod_name, mod) in models)
Serialization.serialize(joinpath(output_path,"gmsc_models.jls"), model_dict)
else
model_dict = Serialization.deserialize(joinpath(output_path,"gmsc_models.jls"))
end
```
```{julia}
params = DataFrame(
Dict(
:n_obs => Int.(round(n_obs/10)*10),
:epochs => epochs,
:batch_size => batch_size,
:n_hidden => n_hidden,
:n_layers => length(model_dict["MLP"].fitresult[1][1])-1,
:activation => string(activation),
:n_ens => n_ens,
:lambda => string(α[3]),
:jem_sampling_steps => jem.sampling_steps,
:sgld_batch_size => sampler.batch_size,
:dataname => "GMSC",
)
)
CSV.write(joinpath(params_path, "gmsc.csv"), params)
```
```{julia}
# Evaluate models:
measure = Dict( :f1score => multiclass_f1score,
:acc => accuracy,
:precision => multiclass_precision
)
model_performance = DataFrame()
for (mod_name, mod) in model_dict
# Test performance:
_perf = CounterfactualExplanations.Models.model_evaluation(mod, test_data, measure=collect(values(measure)))
_perf = DataFrame([[p] for p in _perf], collect(keys(measure)))
_perf.mod_name .= mod_name
_perf.dataname .= "GMSC"
model_performance = vcat(model_performance, _perf)
end
Serialization.serialize(joinpath(output_path,"gmsc_model_performance.jls"), model_performance)
CSV.write(joinpath(output_path, "gmsc_model_performance.csv"), model_performance)
model_performance
```
## Benchmark
```{julia}
λ₁ = 0.1
λ₂ = 0.5
λ₃ = 0.5
Λ = [λ₁, λ₂, λ₃]
opt = Flux.Optimise.Descent(0.05)
use_class_loss = false
# Benchmark generators:
generator_dict = Dict(
"Wachter" => WachterGenerator(λ=λ₁, opt=opt),
"REVISE" => REVISEGenerator(λ=λ₁, opt=opt),
"Schut" => GreedyGenerator(),
"ECCCo" => ECCCoGenerator(
λ=Λ, opt=opt, use_class_loss=use_class_loss,
nsamples=10, nmin=10,
),
)
```
```{julia}
generator_params = DataFrame(
Dict(
:λ1 => λ₁,
:λ2 => λ₂,
:λ3 => λ₃,
:opt => string(typeof(opt)),
:eta => opt.eta,
:dataname => "GMSC",
)
)
CSV.write(joinpath(params_path, "generator/gmsc.csv"), generator_params)
```
```{julia}
# Measures:
measures = [
CounterfactualExplanations.distance,
ECCCo.distance_from_energy,
ECCCo.distance_from_targets,
CounterfactualExplanations.Evaluation.validity,
CounterfactualExplanations.Evaluation.redundancy,
ECCCo.set_size_penalty
]
bmks = []
for target in sort(unique(labels))
for factual in sort(unique(labels))
if factual == target continue
end
bmk = benchmark(
counterfactual_data;
models=model_dict,
generators=generator_dict,
measure=measures,
suppress_training=true, dataname="GMSC",
n_individuals=10,
target=target, factual=factual,
initialization=:identity,
converge_when=:generator_conditions,
)
push!(bmks, bmk)
end
end
bmk = reduce(vcat, bmks)
CSV.write(joinpath(output_path, "gmsc_benchmark.csv"), bmk())
```
```{julia}
df = @chain bmk() begin
@mutate(variable = ifelse.(variable .== "distance_from_energy", "Non-Conformity", variable))
@mutate(variable = ifelse.(variable .== "distance_from_targets", "Implausibility", variable))
@mutate(variable = ifelse.(variable .== "distance", "Cost", variable))
@mutate(variable = ifelse.(variable .== "redundancy", "Redundancy", variable))
@mutate(variable = ifelse.(variable .== "Validity", "Validity", variable))
end
plt = AlgebraOfGraphics.data(df) * visual(BoxPlot) *
mapping(:generator, :value, row=:variable, col=:model, color=:generator)
plt = draw(
plt, axis=(xlabel="", xticksvisible=false, xticklabelsvisible=false, width=150, height=120),
facet=(; linkyaxes=:none)
)
display(plt)
save(joinpath(output_images_path, "gmsc_benchmark.png"), plt, px_per_unit=5)
```