diff --git a/.vscode/settings.json b/.vscode/settings.json
index f848dd07ae86a8a83ef94cfaa2a627725db817ac..7a73a41bfdf76d6f793007240d80983a52f15f97 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,3 +1,2 @@
{
- "julia.environmentPath": "/Users/paltmeyer/code/ECCCo.jl"
}
\ No newline at end of file
diff --git a/experiments/california_housing.jl b/experiments/california_housing.jl
index 99e598a9596a2f9ff84f3bed7e22c1f7a3676f4f..e8a8db16f50a98d467f8cbb9d8b21d9a5b9b3d8d 100644
--- a/experiments/california_housing.jl
+++ b/experiments/california_housing.jl
@@ -44,7 +44,7 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
if GRID_SEARCH
grid_search(
diff --git a/experiments/circles.jl b/experiments/circles.jl
index f07899065ad3757e2fb457b84fd49c3ebe39682e..0d4a6374afeab3df6432dd03d1a4e49339d2e6df 100644
--- a/experiments/circles.jl
+++ b/experiments/circles.jl
@@ -29,7 +29,7 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
if GRID_SEARCH
grid_search(
diff --git a/experiments/daic/testing/lin_sep_final.sh b/experiments/daic/testing/lin_sep_final.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6884d986544d77f7829a0100540176b83e3eed92
--- /dev/null
+++ b/experiments/daic/testing/lin_sep_final.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#SBATCH --job-name="Linearly Separable (ECCCo)"
+#SBATCH --time=00:30:00
+#SBATCH --ntasks=5
+#SBATCH --cpus-per-task=4
+#SBATCH --partition=general
+#SBATCH --mem-per-cpu=2GB
+#SBATCH --mail-type=END # Set mail type to 'END' to receive a mail when the job finishes.
+
+module use /opt/insy/modulefiles # Use DAIC INSY software collection
+module load openmpi
+
+source experiments/slurm_header.sh
+
+srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=linearly_separable output_path=results_testing mpi threaded n_individuals=100 n_runs=2 > experiments/logs/linearly_separable.log
diff --git a/experiments/german_credit.jl b/experiments/german_credit.jl
index 082567b2d6fa123feba3627a378ff44bc31c7fce..6a127b07dc31ed445443c5f9ce170669fd18f4b2 100644
--- a/experiments/german_credit.jl
+++ b/experiments/german_credit.jl
@@ -44,7 +44,7 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
if GRID_SEARCH
grid_search(
diff --git a/experiments/gmsc.jl b/experiments/gmsc.jl
index 3c55d0a585569b8a0712b8afb46489b38b92d721..2b93ff4e5aa4d846f9ae76411e7a470a16d84c4b 100644
--- a/experiments/gmsc.jl
+++ b/experiments/gmsc.jl
@@ -44,7 +44,7 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
if GRID_SEARCH
grid_search(
diff --git a/experiments/grid_search.jl b/experiments/grid_search.jl
index 967cc8a8b5961725c7e3be1059fbf8c372880b0e..fff7c41427728660de991179345a825abf239cff 100644
--- a/experiments/grid_search.jl
+++ b/experiments/grid_search.jl
@@ -56,9 +56,9 @@ function grid_search(
)
# Collect:
- params = map(x -> typeof(x[2]) <: Vector ? x[1] => Tuple(x[2]) : x[1] => x[2], params)
+ _params = map(x -> typeof(x[2]) <: Vector ? x[1] => Tuple(x[2]) : x[1] => x[2], params)
df_params =
- DataFrame(merge(Dict(:id => counter), Dict(params))) |>
+ DataFrame(merge(Dict(:id => counter), Dict(_params))) |>
x -> select(x, :id, Not(:id))
df_outcomes =
DataFrame(Dict(:id => counter, :params => params, :outcome => outcome)) |>
@@ -192,11 +192,6 @@ function best_absolute_outcome(
higher_is_better = [var ∈ ["validity", "redundancy"] for var in evaluation.variable]
evaluation.value[higher_is_better] .= -evaluation.value[higher_is_better]
- # # Normalise to allow for comparison across measures:
- # evaluation =
- # groupby(evaluation, [:dataname, :variable]) |>
- # x -> transform(x, :value => standardize => :value)
-
# Reconstruct outcome with normalised values:
bmk = CounterfactualExplanations.Evaluation.Benchmark(evaluation)
outcome = ExperimentOutcome(exper, model_dict, generator_dict, bmk)
@@ -229,12 +224,19 @@ best_absolute_outcome_eccco(outcomes; kwrgs...) =
best_absolute_outcome_eccco_Δ(outcomes; kwrgs...) =
best_absolute_outcome(outcomes; generator = ECCCo_Δ_NAMES, kwrgs...)
+"""
+ best_outcome(outcomes)
+
+The best outcome is chosen as follows: choose the outcome with the minium average unfaithfulness (`distance_from_energy_l2`) aggregated across all ECCCo generators (`ECCCo_Δ_NAMES`) for the weakest models (`MLP` and `MLP Ensemble`).
+"""
+best_outcome(outcomes; measure=["distance_from_energy_l2"]) = best_absolute_outcome(outcomes; generator=ECCCo_Δ_NAMES, measure=measure, model=["MLP", "MLP Ensemble"])
+
"""
append_best_params!(params::NamedTuple, dataname::String)
Appends the best parameters from grid search results to the specified parameters.
"""
-function append_best_params!(params::NamedTuple, dataname::String)
+function append_best_params(params::NamedTuple, dataname::String)
if !isfile(
joinpath(
DEFAULT_OUTPUT_PATH,
@@ -252,7 +254,10 @@ function append_best_params!(params::NamedTuple, dataname::String)
"$(replace(lowercase(dataname), " " => "_")).jls",
),
)
- best_params = best_absolute_outcome_eccco_Δ(grid_search_results).params
+ best_params = best_outcome(grid_search_results).params
params = (; params..., best_params...)
+
+ params = (; params..., (; Λ = typeof(params.Λ) <: Tuple ? collect(params.Λ) : params.Λ)...)
end
+ return params
end
diff --git a/experiments/linearly_separable.jl b/experiments/linearly_separable.jl
index 2421bd9a9beef61da801b325ebec1a45ab986a2a..43554f3647d8dc92dc5ae03201044faadf84bca9 100644
--- a/experiments/linearly_separable.jl
+++ b/experiments/linearly_separable.jl
@@ -29,7 +29,8 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
+@info "Using the following parameters: $(params)"
if GRID_SEARCH
grid_search(
diff --git a/experiments/moons.jl b/experiments/moons.jl
index a15002795c7a4d135eba54c7c6e8088098fa193d..809bdc406ace3d45f84f85d118561803ccbeebca 100644
--- a/experiments/moons.jl
+++ b/experiments/moons.jl
@@ -28,7 +28,7 @@ params = (
)
# Best grid search params:
-append_best_params!(params, dataname)
+params = append_best_params(params, dataname)
if GRID_SEARCH
grid_search(
diff --git a/experiments/post_processing/hypothesis_tests.jl b/experiments/post_processing/hypothesis_tests.jl
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/experiments/post_processing/results.jl b/experiments/post_processing/results.jl
index 81f77140f85ca6d72491041d9bac2d48414803e1..81d291cb291d0f5cb50bab8e4f5695b5370c0e7e 100644
--- a/experiments/post_processing/results.jl
+++ b/experiments/post_processing/results.jl
@@ -13,7 +13,7 @@ function summarise_outcome(
measure = isnothing(measure) ? unique(bmk().variable) : measure
df = bmk()
# If the :run column is missing (single runs), add it:
- if !(:run ∈ names(df))
+ if !("run" ∈ names(df))
df.run .= 1
end
# Aggregate per run: