diff --git a/experiments/benchmarking/benchmarking.jl b/experiments/benchmarking/benchmarking.jl index 038c2563cfa57614ecea642770a05180b45d474a..fd74c569848cb131c10f3536ef449233f4dd788e 100644 --- a/experiments/benchmarking/benchmarking.jl +++ b/experiments/benchmarking/benchmarking.jl @@ -6,6 +6,7 @@ function default_generators(; opt=Flux.Optimise.Descent(0.01), nsamples::Union{Nothing,Int}=nothing, nmin::Union{Nothing,Int}=nothing, + reg_strength::Real=0.5, ) @info "Begin benchmarking counterfactual explanations." @@ -20,9 +21,9 @@ function default_generators(; "ECCCo" => ECCCoGenerator(λ=Λ, opt=opt, use_class_loss=use_class_loss, nsamples=nsamples, nmin=nmin), "ECCCo (no CP)" => ECCCoGenerator(λ=[λâ‚, 0.0, λ₃], opt=opt, use_class_loss=use_class_loss, nsamples=nsamples, nmin=nmin), "ECCCo (no EBM)" => ECCCoGenerator(λ=[λâ‚, λ₂, 0.0], opt=opt, use_class_loss=use_class_loss, nsamples=nsamples, nmin=nmin), - "ECCCo-Δ" => ECCCoGenerator(λ=Λ_Δ, opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin), - "ECCCo-Δ (no CP)" => ECCCoGenerator(λ=[λâ‚_Δ, 0.0, λ₃_Δ], opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin), - "ECCCo-Δ (no EBM)" => ECCCoGenerator(λ=[λâ‚_Δ, λ₂_Δ, 0.0], opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin), + "ECCCo-Δ" => ECCCoGenerator(λ=Λ_Δ, opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin, reg_strength=reg_strength), + "ECCCo-Δ (no CP)" => ECCCoGenerator(λ=[λâ‚_Δ, 0.0, λ₃_Δ], opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin, reg_strength=reg_strength), + "ECCCo-Δ (no EBM)" => ECCCoGenerator(λ=[λâ‚_Δ, λ₂_Δ, 0.0], opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin, reg_strength=reg_strength), ) else generator_dict = Dict( @@ -30,7 +31,7 @@ function default_generators(; "REVISE" => REVISEGenerator(λ=λâ‚, opt=opt), "Schut" => GreedyGenerator(), "ECCCo" => ECCCoGenerator(λ=Λ, opt=opt, use_class_loss=use_class_loss, nsamples=nsamples, nmin=nmin), - "ECCCo-Δ" => ECCCoGenerator(λ=Λ_Δ, opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin), + "ECCCo-Δ" => ECCCoGenerator(λ=Λ_Δ, opt=opt, use_class_loss=use_class_loss, use_energy_delta=true, nsamples=nsamples, nmin=nmin, reg_strength=reg_strength), ) end return generator_dict @@ -63,6 +64,7 @@ function run_benchmark(exper::Experiment, model_dict::Dict) opt=exper.opt, nsamples=exper.nsamples, nmin=exper.nmin, + reg_strength=exper.reg_strength, ) end diff --git a/experiments/credit_default.jl b/experiments/credit_default.jl new file mode 100644 index 0000000000000000000000000000000000000000..19bbfe2451934d91d1a78dede8701eff0e0d3ce6 --- /dev/null +++ b/experiments/credit_default.jl @@ -0,0 +1,27 @@ +counterfactual_data, test_data = train_test_split(load_credit_default(nothing); test_size=TEST_SIZE) + +# Default builder: +n_hidden = 128 +activation = Flux.swish +builder = MLJFlux.@builder Flux.Chain( + Dense(n_in, n_hidden, activation), + Dense(n_hidden, n_hidden, activation), + Dense(n_hidden, n_out), +) + +# Number of individuals: +n_ind = N_IND_SPECIFIED ? N_IND : 10 + +run_experiment( + counterfactual_data, test_data; + dataname="Credit Default", + builder=builder, + α=[1.0, 1.0, 1e-1], + sampling_batch_size=10, + sampling_steps=30, + use_ensembling=true, + Λ=[0.1, 0.5, 0.5], + opt=Flux.Optimise.Descent(0.05), + n_individuals=n_ind, + use_variants=false +) \ No newline at end of file diff --git a/experiments/experiment.jl b/experiments/experiment.jl index 58efc6c61f126434a0be73c20ed541f601aa29f3..a6b13920a725cda35d0ed76b72a7557ddce6750c 100644 --- a/experiments/experiment.jl +++ b/experiments/experiment.jl @@ -36,6 +36,7 @@ Base.@kwdef struct Experiment finaliser::Function = Flux.softmax loss::Function = Flux.Losses.crossentropy train_parallel::Bool = false + reg_strength::Real = 0.5 end "A container to hold the results of an experiment." diff --git a/experiments/synthetic.sh b/experiments/synthetic.sh index 1ff7acac3f87ea326c2bfc99aab45375beef9893..f1f4b4f10ffd7941565998d40dd14db643c853cb 100644 --- a/experiments/synthetic.sh +++ b/experiments/synthetic.sh @@ -11,4 +11,4 @@ module load 2023r1 openmpi -srun julia --project=experiments experiments/run_experiments.jl -- data=linearly_separable,moons,circles output_path=results retrain threaded mpi > experiments/synthetic.log +srun julia --project=experiments --threads 4 experiments/run_experiments.jl -- data=linearly_separable,moons,circles output_path=results retrain threaded mpi > experiments/synthetic.log diff --git a/src/generator.jl b/src/generator.jl index d5a44ef12c8fb9cdfc0f6f4c34fc6f0ec8482c4a..5a1117b8110cce24aaf88aae8e5e6e7284d7b762 100644 --- a/src/generator.jl +++ b/src/generator.jl @@ -11,6 +11,7 @@ function ECCCoGenerator(; use_energy_delta::Bool=false, nsamples::Union{Nothing,Int}=nothing, nmin::Union{Nothing,Int}=nothing, + reg_strength::Real=0.5, kwargs... ) @@ -30,7 +31,7 @@ function ECCCoGenerator(; end _energy_penalty = - use_energy_delta ? (ECCCo.energy_delta, (n=nsamples, nmin=nmin)) : (ECCCo.distance_from_energy, (n=nsamples, nmin=nmin)) + use_energy_delta ? (ECCCo.energy_delta, (n=nsamples, nmin=nmin, reg_strength=reg_strength)) : (ECCCo.distance_from_energy, (n=nsamples, nmin=nmin)) _penalties = [ (Objectives.distance_l1, []), diff --git a/src/penalties.jl b/src/penalties.jl index aff3e84e5cfb55f578e6006a7911980831642968..78ff7df342e0898571cf91ec9acb2412632001d9 100644 --- a/src/penalties.jl +++ b/src/penalties.jl @@ -46,10 +46,10 @@ function energy_delta( choose_random=false, nmin::Int=25, return_conditionals=false, + reg_strength=0.5, kwargs... ) - _loss = 0.0 nmin = minimum([nmin, n]) @assert choose_lowest_energy ⊻ choose_random || !choose_lowest_energy && !choose_random "Must choose either lowest energy or random samples or neither." @@ -88,7 +88,7 @@ function energy_delta( if return_conditionals return conditional_samples[1] end - return gen_loss + 0.1reg_loss + return gen_loss + reg_strength * reg_loss end