diff --git a/experiments/model_tuning.jl b/experiments/model_tuning.jl new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/experiments/post_processing/results.jl b/experiments/post_processing/results.jl index 1c7578a355e7f8a07595975cd32483a6dc0aac08..eff3de9e41714329489ec9a8f753f490965080ca 100644 --- a/experiments/post_processing/results.jl +++ b/experiments/post_processing/results.jl @@ -3,48 +3,36 @@ Helper function to quickly filter a benchmark table for the distance from targets: the smaller this distance, the higher the plausibility. """ -function plausibility(outcome::ExperimentOutcome) +function summarise_outcome(outcome::ExperimentOutcome; measure="distance_from_targets", model::Union{Nothing,String}=nothing) bmk = outcome.bmk - df = @chain bmk() begin - @group_by(dataname, generator, model, variable) - @summarize(mean=mean(value),sd=std(value)) - @filter(variable == "distance_from_targets") - @ungroup - @arrange(mean) + df = groupby(bmk(), [:dataname, :generator, :model, :variable]) |> + x -> combine(x, :value => mean => :mean, :value => std => :std) |> + x -> subset(x, :variable => ByRow(x -> x ==measure)) + if !isnothing(model) + df = subset(df, :model => ByRow(x -> x == model)) end + sort!(df, [:model, :mean]) return df end +""" + plausibility(outcome::ExperimentOutcome) + +Helper function to quickly filter a benchmark table for the distance from targets: the smaller this distance, the higher the plausibility. +""" +plausibility(outcome::ExperimentOutcome; kwrgs...) = summarise_outcome(outcome, measure="distance_from_targets", kwrgs...) + + """ faithfulness(outcome::ExperimentOutcome) Helper function to quickly filter a benchmark table for the distance from energy: the smaller this distance, the higher the faithfulness. """ -function faithfulness(outcome::ExperimentOutcome) - bmk = outcome.bmk - df = @chain bmk() begin - @group_by(dataname, generator, model, variable) - @summarize(mean=mean(value),sd=std(value)) - @filter(variable == "distance_from_energy") - @ungroup - @arrange(mean) - end - return df -end +faithfulness(outcome::ExperimentOutcome; kwrgs...) = summarise_outcome(outcome, measure="distance_from_energy", kwrgs...) """ closeness(outcome::ExperimentOutcome) Helper function to quickly filter a benchmark table for the distance from the factual: the smaller this distance, the higher the closeness desideratum. """ -function closeness(outcome::ExperimentOutcome) - bmk = outcome.bmk - df = @chain bmk() begin - @group_by(dataname, generator, model, variable) - @summarize(mean=mean(value),sd=std(value)) - @filter(variable == "distance") - @ungroup - @arrange(mean) - end - return df -end \ No newline at end of file +closeness(outcome::ExperimentOutcome; kwrgs...) = summarise_outcome(outcome, measure="distance", kwrgs...) \ No newline at end of file diff --git a/experiments/setup_env.jl b/experiments/setup_env.jl index 18a4438161e99f16b1e7ccfd02b045fdef43827a..5eb01a4fa38df9f31b4f721249b44dea564dd2c5 100644 --- a/experiments/setup_env.jl +++ b/experiments/setup_env.jl @@ -19,7 +19,7 @@ using JointEnergyModels using LazyArtifacts using Logging using Metalhead -using MLJBase: multiclass_f1score, accuracy, multiclass_precision, table, machine, fit! +using MLJBase: multiclass_f1score, accuracy, multiclass_precision, table, machine, fit!, Supervised using MLJEnsembles using MLJFlux using Random @@ -37,6 +37,7 @@ include("experiment.jl") include("grid_search.jl") include("data/data.jl") include("models/models.jl") +include("model_tuning.jl") include("benchmarking/benchmarking.jl") include("post_processing/post_processing.jl") include("utils.jl")