I hate it here

experiments/california_housing.jl

@@ -20,6 +20,9 @@ params = (
     opt=Flux.Optimise.Descent(0.05)
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH
     run_experiment(
         counterfactual_data, test_data;

experiments/circles.jl

@@ -21,6 +21,9 @@ params = (
     sampling_steps=30,
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH
     run_experiment(
         counterfactual_data, test_data;

experiments/german_credit.jl

@@ -20,6 +20,9 @@ params = (
     opt=Flux.Optimise.Descent(0.05)
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH
     run_experiment(
         counterfactual_data, test_data;

experiments/gmsc.jl

@@ -21,6 +21,9 @@ params = (
     opt = Flux.Optimise.Descent(0.05)
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH
     run_experiment(
         counterfactual_data, test_data;

experiments/grid_search.jl

@@ -52,10 +52,22 @@ function grid_search(
     end
 end
 
+const ALL_ECCCO_NAMES = [
+    "ECCCo",
+    "ECCCo (no CP)",
+    "ECCCo (no EBM)",
+    "ECCCo-Δ",
+    "ECCCo-Δ (no CP)",
+    "ECCCo-Δ (no EBM)",
+]
+
 const ECCCO_NAMES = [
     "ECCCo",
     "ECCCo (no CP)",
     "ECCCo (no EBM)",
+]
+
+const ECCCo_Δ_NAMES = [
     "ECCCo-Δ",
     "ECCCo-Δ (no CP)",
     "ECCCo-Δ (no EBM)",
@@ -66,7 +78,7 @@ const ECCCO_NAMES = [
 
 Returns the best outcome from grid search results. The best outcome is defined as the one with the lowest average rank across all datasets and variables for the specified generator and measure.
 """
-function best_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure=["distance_from_energy", "distance_from_targets"], model::Union{Nothing,AbstractArray}=nothing)
+function best_outcome(outcomes::Dict; generator=ALL_ECCCO_NAMES, measure=["distance_from_energy_l2", "distance_from_targets_l2"], model::Union{Nothing,AbstractArray}=nothing)
     ranks = []
     for (params, outcome) in outcomes
         _ranks = generator_rank(outcome; generator=generator, measure=measure, model=model) |>
@@ -82,16 +94,16 @@ function best_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure=["distance_
     return best_outcome
 end
 
-best_eccco(outcomes) = best_outcome(outcomes; generator=["ECCCo"], measure=["distance_from_energy", "distance_from_targets"])
+best_eccco(outcomes) = best_outcome(outcomes; generator=ECCCO_NAMES)
 
-best_eccco_Δ(outcomes) = best_outcome(outcomes; generator=["ECCCo-Δ"], measure=["distance_from_energy", "distance_from_targets"])
+best_eccco_Δ(outcomes) = best_outcome(outcomes; generator=ECCCo_Δ_NAMES)
 
 """
     best_absolute_outcome(outcomes; generator=ECCCO_NAMES, measure="distance_from_energy")
 
 Return the best outcome from grid search results. The best outcome is defined as the one with the lowest average value across all datasets and variables for the specified generator and measure.
 """
-function best_absolute_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure::String="distance_from_energy", model::Union{Nothing,AbstractArray}=nothing)
+function best_absolute_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure::String="distance_from_energy_l2", model::Union{Nothing,AbstractArray}=nothing)
     avg_values = []
     for (params, outcome) in outcomes
         # Compute:
@@ -113,6 +125,21 @@ function best_absolute_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure::S
     )
 end
 
-best_absolute_outcome_eccco(outcomes) = best_absolute_outcome(outcomes; generator=["ECCCo"], measure="distance_from_energy")
+best_absolute_outcome_eccco(outcomes) = best_absolute_outcome(outcomes; generator=ECCCO_NAMES)
 
-best_absolute_outcome_eccco_Δ(outcomes) = best_absolute_outcome(outcomes; generator=["ECCCo-Δ"], measure="distance_from_energy")
\ No newline at end of file
+best_absolute_outcome_eccco_Δ(outcomes) = best_absolute_outcome(outcomes; generator=ECCCo_Δ_NAMES)
+
+"""
+    append_best_params!(params::NamedTuple, dataname::String)
+
+Appends the best parameters from grid search results to the specified parameters.
+"""
+function append_best_params!(params::NamedTuple, dataname::String)
+    if !isfile(joinpath(DEFAULT_OUTPUT_PATH, "grid_search", "$(replace(lowercase(dataname), " " => "_")).jls"))
+        @warn "No grid search results found. Using default parameters."
+    else
+        grid_search_results = Serialization.deserialize(joinpath(DEFAULT_OUTPUT_PATH, "grid_search", "$(replace(lowercase(dataname), " " => "_")).jls"))
+        best_params = best_eccco_Δ(grid_search_results).params
+        params = (; params..., best_params...)
+    end
+end
\ No newline at end of file

experiments/jobscripts/generators/innovation/california_housing.sh

+#!/bin/bash
+
+#SBATCH --job-name="California Housing (ECCCo)"
+#SBATCH --time=3:00:00
+#SBATCH --ntasks=48
+#SBATCH --cpus-per-task=1
+#SBATCH --partition=compute
+#SBATCH --mem-per-cpu=4GB
+#SBATCH --account=innovation
+#SBATCH --mail-type=END     # Set mail type to 'END' to receive a mail when the job finishes. 
+
+module load 2023r1 openmpi
+
+srun julia --project=experiments experiments/run_experiments.jl -- data=california_housing output_path=results mpi > experiments/california_housing.log

experiments/jobscripts/generators/innovation/synthetic.sh

+#!/bin/bash
+
+#SBATCH --job-name="Synthetic (ECCCo)"
+#SBATCH --time=10:00:00
+#SBATCH --ntasks=48
+#SBATCH --cpus-per-task=1
+#SBATCH --partition=compute
+#SBATCH --mem-per-cpu=4GB
+#SBATCH --account=innovation
+#SBATCH --mail-type=END     # Set mail type to 'END' to receive a mail when the job finishes. 
+
+module load 2023r1 openmpi
+
+srun julia --project=experiments experiments/run_experiments.jl -- data=linearly_separable,moons,circles output_path=results mpi > experiments/synthetic.log

experiments/jobscripts/generators/innovation/tabular.sh

+#!/bin/bash
+
+#SBATCH --job-name="Tabular (ECCCo)"
+#SBATCH --time=12:00:00
+#SBATCH --ntasks=48
+#SBATCH --cpus-per-task=1
+#SBATCH --partition=compute
+#SBATCH --mem-per-cpu=4GB
+#SBATCH --account=innovation
+#SBATCH --mail-type=END     # Set mail type to 'END' to receive a mail when the job finishes. 
+
+module load 2023r1 openmpi
+
+srun julia --project=experiments experiments/run_experiments.jl -- data=gmsc,german_credit output_path=results mpi > experiments/tabular.log

experiments/linearly_separable.jl

@@ -22,6 +22,9 @@ params = (
     epochs=100,
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH 
     run_experiment(
         counterfactual_data, test_data; 

experiments/mnist.jl

@@ -45,7 +45,7 @@ params = (
     Λ=[0.1, 0.25, 0.25],
     Λ_Δ=[0.1, 0.1, 1.0],
     opt=Flux.Optimise.Descent(0.1),
-    reg_strength = 0.0,
+    reg_strength = 0.01,
     ce_measures=ce_measures,
 )
 

experiments/moons.jl

@@ -21,6 +21,9 @@ params = (
     sampling_steps=30,
 )
 
+# Best grid search params:
+append_best_params!(params, dataname)
+
 if !GRID_SEARCH
     run_experiment(
         counterfactual_data, test_data;