Merge branch '85-overshooting' of https://github.com/pat-alt/ECCCo.jl into 85-overshooting

.vscode/settings.json

 {
-    "julia.environmentPath": "/Users/paltmeyer/code/ECCCo.jl"
 }
\ No newline at end of file

experiments/california_housing.jl

@@ -44,7 +44,7 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
 
 if GRID_SEARCH
     grid_search(

experiments/circles.jl

@@ -29,7 +29,7 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
 
 if GRID_SEARCH
     grid_search(

experiments/daic/testing/lin_sep_final.sh

+#!/bin/bash
+
+#SBATCH --job-name="Linearly Separable (ECCCo)"
+#SBATCH --time=00:30:00
+#SBATCH --ntasks=5
+#SBATCH --cpus-per-task=4
+#SBATCH --partition=general
+#SBATCH --mem-per-cpu=2GB
+#SBATCH --mail-type=END     # Set mail type to 'END' to receive a mail when the job finishes. 
+
+module use /opt/insy/modulefiles          # Use DAIC INSY software collection
+module load openmpi
+
+source experiments/slurm_header.sh
+
+srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=linearly_separable output_path=results_testing mpi threaded n_individuals=100 n_runs=2 > experiments/logs/linearly_separable.log

experiments/german_credit.jl

@@ -44,7 +44,7 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
 
 if GRID_SEARCH
     grid_search(

experiments/gmsc.jl

@@ -44,7 +44,7 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
 
 if GRID_SEARCH
     grid_search(

experiments/grid_search.jl

@@ -56,9 +56,9 @@ function grid_search(
         )
 
         # Collect:
-        params = map(x -> typeof(x[2]) <: Vector ? x[1] => Tuple(x[2]) : x[1] => x[2], params)
+        _params = map(x -> typeof(x[2]) <: Vector ? x[1] => Tuple(x[2]) : x[1] => x[2], params)
         df_params =
-            DataFrame(merge(Dict(:id => counter), Dict(params))) |>
+            DataFrame(merge(Dict(:id => counter), Dict(_params))) |>
             x -> select(x, :id, Not(:id))
         df_outcomes =
             DataFrame(Dict(:id => counter, :params => params, :outcome => outcome)) |>
@@ -192,11 +192,6 @@ function best_absolute_outcome(
         higher_is_better = [var ∈ ["validity", "redundancy"] for var in evaluation.variable]
         evaluation.value[higher_is_better] .= -evaluation.value[higher_is_better]
 
-        # # Normalise to allow for comparison across measures:
-        # evaluation =
-        #     groupby(evaluation, [:dataname, :variable]) |>
-        #     x -> transform(x, :value => standardize => :value)
-
         # Reconstruct outcome with normalised values:
         bmk = CounterfactualExplanations.Evaluation.Benchmark(evaluation)
         outcome = ExperimentOutcome(exper, model_dict, generator_dict, bmk)
@@ -229,12 +224,19 @@ best_absolute_outcome_eccco(outcomes; kwrgs...) =
 best_absolute_outcome_eccco_Δ(outcomes; kwrgs...) =
     best_absolute_outcome(outcomes; generator = ECCCo_Δ_NAMES, kwrgs...)
 
+"""
+    best_outcome(outcomes)
+
+The best outcome is chosen as follows: choose the outcome with the minium average unfaithfulness (`distance_from_energy_l2`) aggregated across all ECCCo generators (`ECCCo_Δ_NAMES`) for the weakest models (`MLP` and `MLP Ensemble`).
+"""
+best_outcome(outcomes; measure=["distance_from_energy_l2"]) = best_absolute_outcome(outcomes; generator=ECCCo_Δ_NAMES, measure=measure, model=["MLP", "MLP Ensemble"])
+
 """
     append_best_params!(params::NamedTuple, dataname::String)
 
 Appends the best parameters from grid search results to the specified parameters.
 """
-function append_best_params!(params::NamedTuple, dataname::String)
+function append_best_params(params::NamedTuple, dataname::String)
     if !isfile(
         joinpath(
             DEFAULT_OUTPUT_PATH,
@@ -252,7 +254,10 @@ function append_best_params!(params::NamedTuple, dataname::String)
                 "$(replace(lowercase(dataname), " " => "_")).jls",
             ),
         )
-        best_params = best_absolute_outcome_eccco_Δ(grid_search_results).params
+        best_params = best_outcome(grid_search_results).params
         params = (; params..., best_params...)
+        
+        params = (; params..., (; Λ = typeof(params.Λ) <: Tuple ? collect(params.Λ) : params.Λ)...)
     end
+    return params
 end

experiments/linearly_separable.jl

@@ -29,7 +29,8 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
+@info "Using the following parameters: $(params)"
 
 if GRID_SEARCH
     grid_search(

experiments/moons.jl

@@ -28,7 +28,7 @@ params = (
 )
 
 # Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
 
 if GRID_SEARCH
     grid_search(

experiments/post_processing/results.jl

@@ -13,7 +13,7 @@ function summarise_outcome(
     measure = isnothing(measure) ? unique(bmk().variable) : measure
     df = bmk()
     # If the :run column is missing (single runs), add it:
-    if !(:run ∈ names(df))
+    if !("run" ∈ names(df))
         df.run .= 1
     end
     # Aggregate per run: