From 3f61596aaeadb1a2c80f52ca11707a95c0fb9fb5 Mon Sep 17 00:00:00 2001
From: pat-alt <altmeyerpat@gmail.com>
Date: Thu, 19 Oct 2023 16:51:05 +0200
Subject: [PATCH] small fixes

---
 .vscode/settings.json                           |  1 -
 experiments/grid_search.jl                      | 15 +++++++++------
 experiments/post_processing/hypothesis_tests.jl |  0
 experiments/post_processing/results.jl          |  2 +-
 4 files changed, 10 insertions(+), 8 deletions(-)
 create mode 100644 experiments/post_processing/hypothesis_tests.jl

diff --git a/.vscode/settings.json b/.vscode/settings.json
index f848dd07..7a73a41b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,3 +1,2 @@
 {
-    "julia.environmentPath": "/Users/paltmeyer/code/ECCCo.jl"
 }
\ No newline at end of file
diff --git a/experiments/grid_search.jl b/experiments/grid_search.jl
index 967cc8a8..0c11b9f2 100644
--- a/experiments/grid_search.jl
+++ b/experiments/grid_search.jl
@@ -192,11 +192,6 @@ function best_absolute_outcome(
         higher_is_better = [var ∈ ["validity", "redundancy"] for var in evaluation.variable]
         evaluation.value[higher_is_better] .= -evaluation.value[higher_is_better]
 
-        # # Normalise to allow for comparison across measures:
-        # evaluation =
-        #     groupby(evaluation, [:dataname, :variable]) |>
-        #     x -> transform(x, :value => standardize => :value)
-
         # Reconstruct outcome with normalised values:
         bmk = CounterfactualExplanations.Evaluation.Benchmark(evaluation)
         outcome = ExperimentOutcome(exper, model_dict, generator_dict, bmk)
@@ -229,6 +224,13 @@ best_absolute_outcome_eccco(outcomes; kwrgs...) =
 best_absolute_outcome_eccco_Δ(outcomes; kwrgs...) =
     best_absolute_outcome(outcomes; generator = ECCCo_Δ_NAMES, kwrgs...)
 
+"""
+    best_outcome(outcomes)
+
+The best outcome is chosen as follows: choose the outcome with the minium average unfaithfulness (`distance_from_energy_l2`) aggregated across all ECCCo generators (`ECCCo_Δ_NAMES`) for the weakest models (`MLP` and `MLP Ensemble`).
+"""
+best_outcome(outcomes; measure=["distance_from_energy_l2"]) = best_absolute_outcome(outcomes; generator=ECCCo_Δ_NAMES, measure=measure, model=["MLP", "MLP Ensemble"])
+
 """
     append_best_params!(params::NamedTuple, dataname::String)
 
@@ -252,7 +254,8 @@ function append_best_params!(params::NamedTuple, dataname::String)
                 "$(replace(lowercase(dataname), " " => "_")).jls",
             ),
         )
-        best_params = best_absolute_outcome_eccco_Δ(grid_search_results).params
+        best_params = best_outcome(grid_search_results).params
         params = (; params..., best_params...)
+        @info "Best parameters: $(best_params)"
     end
 end
diff --git a/experiments/post_processing/hypothesis_tests.jl b/experiments/post_processing/hypothesis_tests.jl
new file mode 100644
index 00000000..e69de29b
diff --git a/experiments/post_processing/results.jl b/experiments/post_processing/results.jl
index 81f77140..81d291cb 100644
--- a/experiments/post_processing/results.jl
+++ b/experiments/post_processing/results.jl
@@ -13,7 +13,7 @@ function summarise_outcome(
     measure = isnothing(measure) ? unique(bmk().variable) : measure
     df = bmk()
     # If the :run column is missing (single runs), add it:
-    if !(:run ∈ names(df))
+    if !("run" ∈ names(df))
         df.run .= 1
     end
     # Aggregate per run:
-- 
GitLab