= [
models :LogisticRegression,
:FluxModel,
:FluxEnsemble,
]= Flux.Descent(0.01)
opt = Dict(
generators :Greedy=>GreedyGenerator(),
:Generic=>GenericGenerator(opt = opt),
:REVISE=>REVISEGenerator(opt = opt),
:DICE=>DiCEGenerator(opt = opt),
)
3 Experimental Results
This is a supplementary appendix to the research paper Endogenous Macrodynamics in Algorithmic Recourse. It contains all of the experimental results, including those not highlighted in the actual paper. It also links to additional information about the proposed mitigation strategies.
4 Synthetic Data
This notebook was used to run the experiments for the synthetic datasets and can be used to reproduce the results in the paper. In the following we first run the experiments and then generate visualizations and tables.
4.1 Experiment
= 1000
max_obs = load_synthetic(max_obs)
catalogue = [
choices :linearly_separable,
:overlapping,
:circles,
:moons,
]= filter(p -> p[1] in choices, catalogue) data_sets
= set_up_experiments(data_sets,models,generators) experiments
= []
plts for (exp_name, exp_) in experiments
for (M_name, M) in exp_.models
= round(model_evaluation(M, exp_.test_data),digits=2)
score = plot(M, exp_.test_data, title="$exp_name;\n $M_name ($score)")
plt # Errors:
= findall(vec(round.(probs(M, exp_.test_data.X)) .!= exp_.test_data.y))
ids = exp_.test_data.X[:,ids]
x_wrongly_labelled scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
= vcat(plts..., plt)
plts end
end
= plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
plt savefig(plt, joinpath(www_path,"models_test_before.png"))
using AlgorithmicRecourseDynamics.Models: model_evaluation
= []
plts for (exp_name, exp_) in experiments
for (M_name, M) in exp_.models
= round(model_evaluation(M, exp_.train_data),digits=2)
score = plot(M, exp_.train_data, title="$exp_name;\n $M_name ($score)")
plt # Errors:
= findall(vec(round.(probs(M, exp_.train_data.X)) .!= exp_.train_data.y))
ids = exp_.train_data.X[:,ids]
x_wrongly_labelled scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
= vcat(plts..., plt)
plts end
end
= plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
plt savefig(plt, joinpath(www_path,"models_train_before.png"))
= 5
n_evals = 50
n_rounds = Int(round(n_rounds/n_evals))
evaluate_every = 5
n_folds = 100
T = run_experiments(
results
experiments;=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T
save_path
)Serialization.serialize(joinpath(output_path,"results.jls"),results)
= Dict(key => Dict() for (key,val) in results)
plot_dict = 1
fold for (name, res) in results
= res.experiment
exp_ = Dict(key => [] for (key,val) in exp_.generators)
plot_dict[name] = exp_.recourse_systems[fold]
rec_sys = collect(exp_.system_identifiers)
sys_ids = length(rec_sys)
M for m in 1:M
= sys_ids[m]
model_name, generator_name = rec_sys[m].model
M = round(model_evaluation(M, exp_.test_data),digits=2)
score = plot(M, exp_.test_data, title="$name;\n $model_name ($score)")
plt # Errors:
= findall(vec(round.(probs(M, exp_.test_data.X)) .!= exp_.test_data.y))
ids = exp_.test_data.X[:,ids]
x_wrongly_labelled scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
= vcat(plot_dict[name][generator_name], plt)
plot_dict[name][generator_name] end
end
= Dict(key => reduce(vcat, [plots[key] for plots in values(plot_dict)]) for (key, value) in generators)
plot_dict for (name, plts) in plot_dict
= plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
plt savefig(plt, joinpath(www_path,"models_test_after_$(name).png"))
end
using AlgorithmicRecourseDynamics.Models: model_evaluation
= Dict(key => Dict() for (key,val) in results)
plot_dict = 1
fold for (name, res) in results
= res.experiment
exp_ = Dict(key => [] for (key,val) in exp_.generators)
plot_dict[name] = exp_.recourse_systems[fold]
rec_sys = collect(exp_.system_identifiers)
sys_ids = length(rec_sys)
M for m in 1:M
= sys_ids[m]
model_name, generator_name = rec_sys[m].model
M = rec_sys[m].data
data = round(model_evaluation(M, data),digits=2)
score = plot(M, data, title="$name;\n $model_name ($score)")
plt # Errors:
= findall(vec(round.(probs(M, data.X)) .!= data.y))
ids = data.X[:,ids]
x_wrongly_labelled scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
= vcat(plot_dict[name][generator_name], plt)
plot_dict[name][generator_name] end
end
= Dict(key => reduce(vcat, [plots[key] for plots in values(plot_dict)]) for (key, value) in generators)
plot_dict for (name, plts) in plot_dict
= plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
plt savefig(plt, joinpath(www_path,"models_train_after_$(name).png"))
end
4.2 Plots
= Serialization.deserialize(joinpath(output_path,"results.jls")); results
using Images
= Dict()
line_charts = Dict()
errorbar_charts for (data_name, res) in results
= plot(res)
plt save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
Images.= plt
line_charts[data_name] = plot(res,maximum(res.output.n))
plt save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
Images.= plt
errorbar_charts[data_name] end
4.2.1 Line Charts
Figure 4.1 shows the evolution of the evaluation metrics over the course of the experiment.
4.2.2 Error Bar Charts
Figure 4.2 shows the evaluation metrics at the end of the experiments.
4.3 Bootstrap
= 100
n_bootstrap = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap.csv")) df
┌──────────┬─────────┬────────────────────┬────────────────────┬───────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String31 │ String31 │ String7 │ Float64 │
├──────────┼─────────┼────────────────────┼────────────────────┼───────────┼──────────────┤
│ mmd │ domain │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ REVISE │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ REVISE │ 0.238 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ REVISE │ 0.188 │
│ mmd │ domain │ linearly_separable │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ REVISE │ 0.158 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Generic │ 0.104 │
│ mmd │ domain │ circles │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ DICE │ 0.1275 │
│ mmd │ domain │ circles │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ REVISE │ 0.996 │
│ mmd │ domain │ circles │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ REVISE │ 1.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ REVISE │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ Greedy │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ REVISE │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ DICE │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ REVISE │ 0.084 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ linearly_separable │ FluxModel │ Generic │ 0.87 │
│ mmd │ model │ linearly_separable │ FluxModel │ Greedy │ 0.616 │
│ mmd │ model │ linearly_separable │ FluxModel │ REVISE │ 0.65 │
│ mmd │ model │ linearly_separable │ FluxModel │ DICE │ 0.82 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.892 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Greedy │ 0.528 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ REVISE │ 0.642 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ DICE │ 0.878 │
│ mmd │ model │ circles │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ circles │ FluxModel │ Generic │ 0.392 │
│ mmd │ model │ circles │ FluxModel │ Greedy │ 0.078 │
│ mmd │ model │ circles │ FluxModel │ REVISE │ 0.836 │
│ mmd │ model │ circles │ FluxModel │ DICE │ 0.516 │
│ mmd │ model │ circles │ FluxEnsemble │ Generic │ 0.456 │
│ mmd │ model │ circles │ FluxEnsemble │ Greedy │ 0.16 │
│ mmd │ model │ circles │ FluxEnsemble │ REVISE │ 0.872 │
│ mmd │ model │ circles │ FluxEnsemble │ DICE │ 0.572 │
│ mmd │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ moons │ FluxModel │ Generic │ 0.716 │
│ mmd │ model │ moons │ FluxModel │ Greedy │ 0.17 │
│ mmd │ model │ moons │ FluxModel │ REVISE │ 0.67 │
│ mmd │ model │ moons │ FluxModel │ DICE │ 0.588 │
│ mmd │ model │ moons │ FluxEnsemble │ Generic │ 0.634 │
│ mmd │ model │ moons │ FluxEnsemble │ Greedy │ 0.092 │
│ mmd │ model │ moons │ FluxEnsemble │ REVISE │ 0.75 │
│ mmd │ model │ moons │ FluxEnsemble │ DICE │ 0.61 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxModel │ REVISE │ 0.004 │
│ mmd_grid │ model │ overlapping │ FluxModel │ DICE │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ REVISE │ 0.024 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ DICE │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ REVISE │ 0.018 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ DICE │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ REVISE │ 0.014 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ DICE │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ REVISE │ 0.152 │
│ mmd_grid │ model │ circles │ FluxModel │ DICE │ 0.004 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ REVISE │ 0.14 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ DICE │ 0.002 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ moons │ FluxModel │ Generic │ 0.004 │
│ mmd_grid │ model │ moons │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ moons │ FluxModel │ REVISE │ 0.074 │
│ mmd_grid │ model │ moons │ FluxModel │ DICE │ 0.016 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Generic │ 0.002 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ REVISE │ 0.044 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ DICE │ 0.0 │
└──────────┴─────────┴────────────────────┴────────────────────┴───────────┴──────────────┘
4.4 Chart in paper
Figure 4.3 shows the chart that went into the paper.
load(joinpath(www_artifact_path,"paper_synthetic_results.png")) Images.
# echo: false
generate_artifacts(output_path)
generate_artifacts(www_path)
5 Real-World Data
= [
models :LogisticRegression,
:FluxModel,
:FluxEnsemble
]= Flux.Descent(0.01)
opt = Dict(
generators :Greedy=>GreedyGenerator(),
:Generic=>GenericGenerator(opt = opt),
:REVISE=>REVISEGenerator(opt = opt),
:DICE=>DiCEGenerator(opt = opt),
)
= 5000
max_obs = load_real_world(max_obs)
data_sets = [
choices :cal_housing,
:credit_default,
:gmsc,
]= filter(p -> p[1] in choices, data_sets) data_sets
using CounterfactualExplanations.DataPreprocessing: unpack
= 500
bs function data_loader(data::CounterfactualData)
= unpack(data)
X, y = Flux.DataLoader((X,y),batchsize=bs)
data return data
end
= (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.1) model_params
= set_up_experiments(
experiments
data_sets,models,generators; =100, model_params=model_params,
pre_train_models=data_loader
data_loader )
5.1 Experiment
= 5
n_evals = 50
n_rounds = Int(round(n_rounds/n_evals))
evaluate_every = 5
n_folds = 10000
n_samples = 100
T = (epochs=250, latent_dim=8)
generative_model_params = run_experiments(
results
experiments;=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
save_path=generative_model_params
generative_model_params
)Serialization.serialize(joinpath(output_path,"results.jls"),results)
5.1.1 Plots
= Serialization.deserialize(joinpath(output_path,"results.jls")) results
using Images
= Dict()
line_charts = Dict()
errorbar_charts for (data_name, res) in results
= plot(res)
plt save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
Images.= plt
line_charts[data_name] = plot(res,maximum(res.output.n))
plt save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
Images.= plt
errorbar_charts[data_name] end
5.1.2 Line Charts
Figure 5.1 shows the evolution of the evaluation metrics over the course of the experiment.
= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart")]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
5.1.3 Error Bar Charts
Figure 5.2 shows the evaluation metrics at the end of the experiments.
= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart")]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
5.2 Bootstrap
= 100
n_bootstrap = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap.csv")) df
┌──────────┬─────────┬────────────────┬────────────────────┬───────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String15 │ String31 │ String7 │ Float64 │
├──────────┼─────────┼────────────────┼────────────────────┼───────────┼──────────────┤
│ mmd │ domain │ credit_default │ LogisticRegression │ Generic │ 0.594 │
│ mmd │ domain │ credit_default │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ DICE │ 0.268 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Greedy │ 0.388 │
│ mmd │ domain │ credit_default │ FluxModel │ Generic │ 0.668 │
│ mmd │ domain │ credit_default │ FluxModel │ REVISE │ 0.0 │
│ mmd │ domain │ credit_default │ FluxModel │ DICE │ 0.466 │
│ mmd │ domain │ credit_default │ FluxModel │ Greedy │ 0.998 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Generic │ 0.738 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ DICE │ 0.634 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Greedy │ 1.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ REVISE │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ REVISE │ 0.112 │
│ mmd │ domain │ gmsc │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ REVISE │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ DICE │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Greedy │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ REVISE │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ DICE │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Greedy │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ REVISE │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ DICE │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Greedy │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ REVISE │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ DICE │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Greedy │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ REVISE │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ DICE │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Greedy │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ DICE │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ REVISE │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ DICE │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ DICE │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ REVISE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ DICE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ DICE │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Greedy │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ REVISE │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ DICE │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Greedy │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Generic │ 0.06 │
│ mmd_grid │ model │ gmsc │ FluxModel │ REVISE │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ DICE │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Greedy │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ REVISE │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ DICE │ 0.002 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Greedy │ 0.0 │
└──────────┴─────────┴────────────────┴────────────────────┴───────────┴──────────────┘
5.2.1 Chart in paper
Figure 5.3 shows the chart that went into the paper.
using DataFrames, Statistics
= :FluxEnsemble
model_ = DataFrame()
df for (key, val) in results
= deepcopy(val.output)
df_ .= key
df_.dataset = vcat(df,df_)
df end
= df[df.n .== maximum(df.n),:]
df = df[df.model .== model_,:]
df filter!(:value => x -> !any(f -> f(x), (ismissing, isnothing, isnan)), df)
= groupby(df, [:generator, :dataset, :n, :name, :scope])
gdf = combine(gdf, :value => (x -> [(mean(x),mean(x)+std(x),mean(x)-std(x))]) => [:mean, :ymax, :ymin])
df_plot = df_plot[[name in [:mmd, :model_performance] for name in df_plot.name],:]
df_plot = mapcols(x -> typeof(x) == Vector{Symbol} ? string.(x) : x, df_plot)
df_plot .= [r[:name] == "mmd" ? "$(r[:name])_$(r[:scope])" : r[:name] for r in eachrow(df_plot)]
df_plot.name transform!(df_plot, :dataset => (X -> [x=="cal_housing" ? "California Housing" : x for x in X]) => :dataset)
transform!(df_plot, :dataset => (X -> [x=="credit_default" ? "Credit Default" : x for x in X]) => :dataset)
transform!(df_plot, :dataset => (X -> [x=="gmsc" ? "GMSC" : x for x in X]) => :dataset)
transform!(df_plot, :name => (X -> [x=="mmd_domain" ? "MMD (domain)" : x for x in X]) => :name)
transform!(df_plot, :name => (X -> [x=="mmd_model" ? "MMD (model)" : x for x in X]) => :name)
transform!(df_plot, :name => (X -> [x=="model_performance" ? "Performance" : x for x in X]) => :name)
transform!(df_plot, :generator => (X -> [x=="REVISE" ? "Latent" : x for x in X]) => :generator)
= length(unique(df_plot.dataset))
ncol = length(unique(df_plot.name))
nrow
using RCall
= 1.75
scale_ """
Rlibrary(ggplot2)
plt <- ggplot($df_plot) +
geom_bar(aes(x=n, y=mean, fill=generator), stat="identity", alpha=0.5, position="dodge") +
geom_pointrange( aes(x=n, y=mean, ymin=ymin, ymax=ymax, colour=generator), alpha=0.9, position=position_dodge(width=0.9), size=0.5) +
facet_grid(
rows = vars(name),
cols = vars(dataset),
scales = "free_y"
) +
labs(y = "Value") +
scale_fill_discrete(name="Generator:") +
scale_colour_discrete(name="Generator:") +
theme(
axis.title.x=element_blank(),
axis.text.x=element_blank(),
axis.ticks.x=element_blank(),
legend.position="bottom"
)
temp_path <- file.path(tempdir(), "plot.png")
ggsave(temp_path,width=$ncol * $scale_,height=$nrow * $scale_ * 0.8)
"""
= Images.load(rcopy(R"temp_path"))
img save(joinpath(www_path,"paper_real_world_results.png"), img) Images.
load(joinpath(www_artifact_path,"paper_real_world_results.png")) Images.
6 Mitigation Strategies
= [
models :LogisticRegression,
:FluxModel,
:FluxEnsemble,
]= Flux.Descent(0.01)
opt = Dict(
generators :Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent=>REVISEGenerator(opt = opt),
:Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)
6.1 Synthetic
= 1000
max_obs = load_synthetic(max_obs)
catalogue = [
choices :linearly_separable,
:overlapping,
:circles,
:moons,
]= filter(p -> p[1] in choices, catalogue) data_sets
= set_up_experiments(data_sets,models,generators) experiments
= 5
n_evals = 50
n_rounds = Int(round(n_rounds/n_evals))
evaluate_every = 5
n_folds = 100
T using Serialization
= run_experiments(
results
experiments;=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T
save_path
)Serialization.serialize(joinpath(output_path,"results_synthetic.jls"),results)
6.2 Plots
using Serialization
= Serialization.deserialize(joinpath(output_path,"results_synthetic.jls")) results
using Images
= Dict()
line_charts = Dict()
errorbar_charts for (data_name, res) in results
= plot(res)
plt save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
Images.= plt
line_charts[data_name] = plot(res,maximum(res.output.n))
plt save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
Images.= plt
errorbar_charts[data_name] end
6.2.1 Line Charts
Figure 6.1 shows the evolution of the evaluation metrics over the course of the experiment.
= [
choices :linearly_separable,
:overlapping,
:circles,
:moons,
]= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.2.2 Error Bar Charts
Figure 6.2 shows the evaluation metrics at the end of the experiments.
= [
choices :linearly_separable,
:overlapping,
:circles,
:moons,
]= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.3 Bootstrap
= 100
n_bootstrap = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_synthetic.csv")) df
┌──────────┬─────────┬────────────────────┬────────────────────┬──────────────────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String31 │ String31 │ String31 │ Float64 │
├──────────┼─────────┼────────────────────┼────────────────────┼──────────────────────┼──────────────┤
│ mmd │ domain │ overlapping │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Gravitational │ 0.184 │
│ mmd │ domain │ linearly_separable │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Latent │ 0.358 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Gravitational │ 0.27 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Latent │ 0.586 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.152 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Latent │ 0.38 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ circles │ FluxModel │ Latent │ 1.0 │
│ mmd │ domain │ circles │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Latent │ 1.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ Gravitational │ 0.024 │
│ mmd │ model │ overlapping │ FluxModel │ ClapROAR │ 0.012 │
│ mmd │ model │ overlapping │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Gravitational │ 0.006 │
│ mmd │ model │ overlapping │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Gravitational │ 0.018 │
│ mmd │ model │ overlapping │ FluxEnsemble │ ClapROAR │ 0.004 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ linearly_separable │ FluxModel │ Generic_conservative │ 0.852 │
│ mmd │ model │ linearly_separable │ FluxModel │ Gravitational │ 0.89 │
│ mmd │ model │ linearly_separable │ FluxModel │ ClapROAR │ 0.84 │
│ mmd │ model │ linearly_separable │ FluxModel │ Latent │ 0.722 │
│ mmd │ model │ linearly_separable │ FluxModel │ Generic │ 0.83 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.176 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Gravitational │ 0.898 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.866 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Latent │ 0.338 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.002 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.868 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.912 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.916 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Latent │ 0.822 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.818 │
│ mmd │ model │ circles │ FluxModel │ Generic_conservative │ 0.356 │
│ mmd │ model │ circles │ FluxModel │ Gravitational │ 0.48 │
│ mmd │ model │ circles │ FluxModel │ ClapROAR │ 0.484 │
│ mmd │ model │ circles │ FluxModel │ Latent │ 0.802 │
│ mmd │ model │ circles │ FluxModel │ Generic │ 0.482 │
│ mmd │ model │ circles │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ circles │ FluxEnsemble │ Generic_conservative │ 0.462 │
│ mmd │ model │ circles │ FluxEnsemble │ Gravitational │ 0.282 │
│ mmd │ model │ circles │ FluxEnsemble │ ClapROAR │ 0.534 │
│ mmd │ model │ circles │ FluxEnsemble │ Latent │ 0.91 │
│ mmd │ model │ circles │ FluxEnsemble │ Generic │ 0.408 │
│ mmd │ model │ moons │ FluxModel │ Generic_conservative │ 0.568 │
│ mmd │ model │ moons │ FluxModel │ Gravitational │ 0.868 │
│ mmd │ model │ moons │ FluxModel │ ClapROAR │ 0.594 │
│ mmd │ model │ moons │ FluxModel │ Latent │ 0.598 │
│ mmd │ model │ moons │ FluxModel │ Generic │ 0.604 │
│ mmd │ model │ moons │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ moons │ FluxEnsemble │ Generic_conservative │ 0.48 │
│ mmd │ model │ moons │ FluxEnsemble │ Gravitational │ 0.922 │
│ mmd │ model │ moons │ FluxEnsemble │ ClapROAR │ 0.584 │
│ mmd │ model │ moons │ FluxEnsemble │ Latent │ 0.654 │
│ mmd │ model │ moons │ FluxEnsemble │ Generic │ 0.412 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Generic_conservative │ 0.002 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Gravitational │ 0.034 │
│ mmd_grid │ model │ overlapping │ FluxModel │ ClapROAR │ 0.022 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Latent │ 0.008 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ ClapROAR │ 0.002 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.006 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Gravitational │ 0.034 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ ClapROAR │ 0.058 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Latent │ 0.016 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Gravitational │ 0.062 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Gravitational │ 0.004 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.128 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ Gravitational │ 0.002 │
│ mmd_grid │ model │ circles │ FluxModel │ ClapROAR │ 0.004 │
│ mmd_grid │ model │ circles │ FluxModel │ Latent │ 0.176 │
│ mmd_grid │ model │ circles │ FluxModel │ Generic │ 0.008 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ ClapROAR │ 0.002 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Latent │ 0.24 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ moons │ FluxModel │ Gravitational │ 0.052 │
│ mmd_grid │ model │ moons │ FluxModel │ ClapROAR │ 0.002 │
│ mmd_grid │ model │ moons │ FluxModel │ Latent │ 0.024 │
│ mmd_grid │ model │ moons │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Gravitational │ 0.164 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Latent │ 0.058 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Generic │ 0.0 │
└──────────┴─────────┴────────────────────┴────────────────────┴──────────────────────┴──────────────┘
6.4 Chart in paper
Figure 6.3 shows the chart that went into the paper.
load(joinpath(www_artifact_path,"paper_synthetic_results.png")) Images.
6.5 Latent Space Search
= Dict(
generators :Latent=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)
= set_up_experiments(data_sets,models,generators) experiments
= 5
n_evals = 50
n_rounds = Int(round(n_rounds/n_evals))
evaluate_every = 5
n_folds = 100
T using Serialization
= run_experiments(
results
experiments;=output_path, evaluate_every=evaluate_every, n_rounds=n_rounds, n_folds=n_folds, T=T,
save_path= true
latent_space
)Serialization.serialize(joinpath(output_path,"results_synthetic_latent.jls"),results)
using Serialization
= Serialization.deserialize(joinpath(output_path,"results_synthetic_latent.jls")) results
using Images
= Dict()
line_charts = Dict()
errorbar_charts for (data_name, res) in results
= plot(res)
plt save(joinpath(www_path, "line_chart_latent_$(data_name).png"), plt)
Images.= plt
line_charts[data_name] = plot(res,maximum(res.output.n))
plt save(joinpath(www_path, "errorbar_chart_latent_$(data_name).png"), plt)
Images.= plt
errorbar_charts[data_name] end
6.6 Plots
6.6.1 Line Charts
Figure 6.4 shows the evolution of the evaluation metrics over the course of the experiment.
= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.6.2 Error Bar Charts
Figure 6.5 shows the evaluation metrics at the end of the experiments.
= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.7 Bootstrap
= 100
n_bootstrap = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_latent.csv")) df
6.8 Chart in paper
Figure 6.6 shows the chart that went into the paper.
load(joinpath(www_artifact_path,"paper_synthetic_latent_results.png")) Images.
6.9 Real World
= [
models :LogisticRegression,
:FluxModel,
:FluxEnsemble,
]= Flux.Descent(0.01)
opt = Dict(
generators :Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent=>REVISEGenerator(opt = opt),
:Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)
= 2500
max_obs = data_dir("real_world")
data_path = load_real_world(max_obs)
data_sets = [
choices :cal_housing,
:credit_default,
:gmsc,
]= filter(p -> p[1] in choices, data_sets) data_sets
using CounterfactualExplanations.DataPreprocessing: unpack
= 500
bs function data_loader(data::CounterfactualData)
= unpack(data)
X, y = Flux.DataLoader((X,y),batchsize=bs)
data return data
end
= (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.1) model_params
= set_up_experiments(
experiments
data_sets,models,generators; =100, model_params=model_params,
pre_train_models=data_loader
data_loader )
= 5
n_evals = 50
n_rounds = Int(round(n_rounds/n_evals))
evaluate_every = 5
n_folds = 10000
n_samples = 100
T = (epochs=250, latent_dim=8)
generative_model_params = run_experiments(
results
experiments;=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
save_path=generative_model_params
generative_model_params
)Serialization.serialize(joinpath(output_path,"results_real_world.jls"),results)
using Serialization
= Serialization.deserialize(joinpath(output_path,"results_real_world.jls")) results
using Images
= Dict()
line_charts = Dict()
errorbar_charts for (data_name, res) in results
= plot(res)
plt save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
Images.= plt
line_charts[data_name] = plot(res,maximum(res.output.n))
plt save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
Images.= plt
errorbar_charts[data_name] end
6.9.1 Line Charts
Figure 5.1 shows the evolution of the evaluation metrics over the course of the experiment.
= [
choices :cal_housing,
:credit_default,
:gmsc,
]= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.9.2 Error Bar Charts
Figure 5.2 shows the evaluation metrics at the end of the experiments.
= [
choices :cal_housing,
:credit_default,
:gmsc,
]= readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
img_files for img in img_files
display(load(img))
end
6.9.3 Bootstrap
= 100
n_bootstrap = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_real_world.csv")) df
┌──────────┬─────────┬────────────────┬────────────────────┬──────────────────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String15 │ String31 │ String31 │ Float64 │
├──────────┼─────────┼────────────────┼────────────────────┼──────────────────────┼──────────────┤
│ mmd │ domain │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Generic_conservative │ 1.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Generic │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Generic_conservative │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Generic │ 1.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Generic_conservative │ 0.256 │
│ mmd │ domain │ credit_default │ LogisticRegression │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Generic │ 1.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Latent │ 0.398 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Latent │ 0.204 │
│ mmd │ domain │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Latent │ 0.2 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Gravitational │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Latent │ 0.016 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.008 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Latent │ 0.072 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Generic_conservative │ 0.2 │
│ mmd_grid │ model │ gmsc │ FluxModel │ ClapROAR │ 0.244 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Gravitational │ 0.056 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Generic │ 0.23 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
└──────────┴─────────┴────────────────┴────────────────────┴──────────────────────┴──────────────┘
6.9.4 Chart in paper
Figure 6.9 shows the chart that went into the paper.
load(joinpath(www_artifact_path,"paper_real_world_results.png")) Images.