mnist

experiments/daic/tuning/generators/mnist.sh

@@ -2,10 +2,10 @@
 
 #SBATCH --job-name="MNIST Grid-search (ECCCo)"
 #SBATCH --time=02:00:00
-#SBATCH --ntasks=30
+#SBATCH --ntasks=40
 #SBATCH --cpus-per-task=10
 #SBATCH --partition=general
-#SBATCH --mem-per-cpu=6GB
+#SBATCH --mem-per-cpu=4GB
 #SBATCH --mail-type=END     # Set mail type to 'END' to receive a mail when the job finishes. 
 
 module use /opt/insy/modulefiles          # Use DAIC INSY software collection
@@ -13,4 +13,4 @@ module load openmpi
 
 source experiments/slurm_header.sh
 
-srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=mnist output_path=results mpi grid_search threaded n_individuals=10 n_each=10 > experiments/logs/grid_search_mnist.log
+srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=mnist output_path=results mpi grid_search threaded n_individuals=1 n_each=10 > experiments/logs/grid_search_mnist.log

experiments/post_processing/results.jl

@@ -11,12 +11,23 @@ function summarise_outcome(
 
     bmk = outcome.bmk
     measure = isnothing(measure) ? unique(bmk().variable) : measure
-
+    df = bmk()
+    # If the :run column is missing (single runs), add it:
+    if !(:run ∈ names(df))
+        df.run .= 1
+    end
+    # Aggregate per run:
     df =
-        groupby(bmk(), [:dataname, :generator, :model, :variable]) |>
+        groupby(df, [:dataname, :generator, :model, :run, :variable]) |>
         x ->
-            combine(x, :value => mean => :mean, :value => std => :std_wg) |>
+            combine(x, :value => mean => :mean_group, :value => std => :std_group) |>
             x -> subset(x, :variable => ByRow(x -> x ∈ measure))
+    # Compute mean and std across runs:
+    df =
+        groupby(df, [:dataname, :generator, :model, :variable]) |>
+        x ->
+            combine(x, :mean_group => mean => :mean, :mean_group => std => :std) 
+    # Subset:
     if !isnothing(model)
         df = subset(df, :model => ByRow(x -> x ∈ model))
     end