From 74459fa82b6ed011ca3946eb2c5cfe89f146c673 Mon Sep 17 00:00:00 2001
From: Pat Alt <55311242+pat-alt@users.noreply.github.com>
Date: Mon, 18 Sep 2023 14:41:48 +0200
Subject: [PATCH] best absolute outcome
---
experiments/grid_search.jl | 35 +++++++++++++++++++++++++++++++++--
1 file changed, 33 insertions(+), 2 deletions(-)
diff --git a/experiments/grid_search.jl b/experiments/grid_search.jl
index 07c56e8c..bfb20a3f 100644
--- a/experiments/grid_search.jl
+++ b/experiments/grid_search.jl
@@ -71,7 +71,7 @@ function best_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure=["distance_
for (params, outcome) in outcomes
_ranks = generator_rank(outcome; generator=generator, measure=measure) |>
x -> x.avg_rank |>
- x -> sum(x) / length(x)[1]
+ x -> (sum(x) / length(x))[1]
push!(ranks, _ranks)
end
best_index = argmin(ranks)
@@ -84,4 +84,35 @@ end
best_eccco(outcomes) = best_outcome(outcomes; generator=["ECCCo"], measure=["distance_from_energy", "distance_from_targets"])
-best_eccco_Δ(outcomes) = best_outcome(outcomes; generator=["ECCCo-Δ"], measure=["distance_from_energy", "distance_from_targets"])
\ No newline at end of file
+best_eccco_Δ(outcomes) = best_outcome(outcomes; generator=["ECCCo-Δ"], measure=["distance_from_energy", "distance_from_targets"])
+
+"""
+ best_absolute_outcome(outcomes; generator=ECCCO_NAMES, measure="distance_from_energy")
+
+Return the best outcome from grid search results. The best outcome is defined as the one with the lowest average value across all datasets and variables for the specified generator and measure.
+"""
+function best_absolute_outcome(outcomes::Dict; generator=ECCCO_NAMES, measure::String="distance_from_energy")
+ avg_values = []
+ for (params, outcome) in outcomes
+ # Compute:
+ results = summarise_outcome(outcome, measure=[measure])
+ # Adjust variables for which higher is better:
+ higher_is_better = [var ∈ ["validity", "redundancy"] for var in results.variable]
+ results.mean[higher_is_better] .= -results.mean[higher_is_better]
+ # Compute avergaes:
+ _avg_values = subset(results, :generator => ByRow(x -> x ∈ generator)) |>
+ x -> x.mean |>
+ x -> (sum(x)/length(x))[1]
+ push!(avg_values, _avg_values)
+ end
+ println(avg_values)
+ best_index = argmin(avg_values)
+ best_outcome = (
+ params = collect(keys(outcomes))[best_index],
+ outcome = collect(values(outcomes))[best_index]
+ )
+end
+
+best_absolute_outcome_eccco(outcomes) = best_absolute_outcome(outcomes; generator=["ECCCo"], measure="distance_from_energy")
+
+best_absolute_outcome_eccco_Δ(outcomes) = best_absolute_outcome(outcomes; generator=["ECCCo-Δ"], measure="distance_from_energy")
\ No newline at end of file
--
GitLab