trying with vertical splitting

experiments/benchmarking/benchmarking.jl

@@ -176,6 +176,7 @@ function run_benchmark(exper::Experiment, model_dict::Dict)
         parallelizer = parallelizer,
         store_ce = exper.store_ce,
         n_runs = exper.n_runs,
+        vertical_splits = VERTICAL_SPLITS,
     )
     return bmk, generator_dict
 end

experiments/jobscripts/testing/mnist.sh

@@ -13,4 +13,4 @@ module load 2023r1 openmpi
 
 source experiments/slurm_header.sh
 
-srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=mnist output_path=results mpi threaded n_individuals=5 n_runs=5 n_each=16 > experiments/logs/mnist.log
+srun julia --project=experiments --threads $SLURM_CPUS_PER_TASK experiments/run_experiments.jl -- data=mnist output_path=results mpi threaded n_individuals=20 n_runs=5 vertical_splits=10 > experiments/logs/mnist.log

experiments/setup_env.jl

@@ -220,3 +220,17 @@ STORE_CE = "store_ce" ∈ ARGS
 
 "Boolean flag to chech if best outcome from grid search should be used."
 FROM_GRID_SEARCH = "from_grid" ∈ ARGS
+
+# Vertical splits for benchmarking:
+if any(contains.(ARGS, "vertical_splits"))
+    @assert sum(contains.(ARGS, "output_path")) == 1 "`vertical_splits` is specified more than once."
+    n_splits =
+        ARGS[findall(contains.(ARGS, "output_path"))][1] |>
+        x -> replace(x, "output_path=" => "") |>
+        x -> parse(Int, x)
+else
+    n_splits = nothing
+end
+
+"Number of vertical splits."
+VERTICAL_SPLITS = n_splits