diff --git a/experiments/post_processing/post_processing.jl b/experiments/post_processing/post_processing.jl index 7eefa387067dc6ebfa88b596d5d20d9d1e0aabe2..f70aabe43c34d7474afc522dd98013210eab5543 100644 --- a/experiments/post_processing/post_processing.jl +++ b/experiments/post_processing/post_processing.jl @@ -1,2 +1,3 @@ include("meta_data.jl") -include("artifacts.jl") \ No newline at end of file +include("artifacts.jl") +include("results.jl") \ No newline at end of file diff --git a/experiments/post_processing/results.jl b/experiments/post_processing/results.jl new file mode 100644 index 0000000000000000000000000000000000000000..1c7578a355e7f8a07595975cd32483a6dc0aac08 --- /dev/null +++ b/experiments/post_processing/results.jl @@ -0,0 +1,50 @@ +""" + plausibility(outcome::ExperimentOutcome) + +Helper function to quickly filter a benchmark table for the distance from targets: the smaller this distance, the higher the plausibility. +""" +function plausibility(outcome::ExperimentOutcome) + bmk = outcome.bmk + df = @chain bmk() begin + @group_by(dataname, generator, model, variable) + @summarize(mean=mean(value),sd=std(value)) + @filter(variable == "distance_from_targets") + @ungroup + @arrange(mean) + end + return df +end + +""" + faithfulness(outcome::ExperimentOutcome) + +Helper function to quickly filter a benchmark table for the distance from energy: the smaller this distance, the higher the faithfulness. +""" +function faithfulness(outcome::ExperimentOutcome) + bmk = outcome.bmk + df = @chain bmk() begin + @group_by(dataname, generator, model, variable) + @summarize(mean=mean(value),sd=std(value)) + @filter(variable == "distance_from_energy") + @ungroup + @arrange(mean) + end + return df +end + +""" + closeness(outcome::ExperimentOutcome) + +Helper function to quickly filter a benchmark table for the distance from the factual: the smaller this distance, the higher the closeness desideratum. +""" +function closeness(outcome::ExperimentOutcome) + bmk = outcome.bmk + df = @chain bmk() begin + @group_by(dataname, generator, model, variable) + @summarize(mean=mean(value),sd=std(value)) + @filter(variable == "distance") + @ungroup + @arrange(mean) + end + return df +end \ No newline at end of file diff --git a/experiments/setup_env.jl b/experiments/setup_env.jl index fc9217a1b0a9ad4c01c32846f1834370583ff9c1..b9fda88272eedf2dd8a09d718a58c14318ea46c2 100644 --- a/experiments/setup_env.jl +++ b/experiments/setup_env.jl @@ -25,6 +25,8 @@ using Random using Serialization using TidierData +import MPI + Random.seed!(2023) ENV["DATADEPS_ALWAYS_ACCEPT"] = "true" # avoid command prompt and just download data @@ -48,7 +50,6 @@ else end if "mpi" ∈ ARGS - import MPI MPI.Init() const USE_MPI = true plz = MPIParallelizer(MPI.COMM_WORLD; threaded=USE_THREADS)