another attempt

experiments/mnist.sh

@@ -11,4 +11,4 @@
 
 module load 2023r1 openmpi
 
-srun julia --project=experiments experiments/run_experiments.jl -- data=mnist output_path=results mpi > experiments/mnist.log
+srun julia --project=experiments experiments/run_experiments.jl -- data=mnist output_path=results mpi retrain > experiments/mnist.log

experiments/models/models.jl

@@ -47,20 +47,9 @@ function prepare_models(exper::Experiment)
         @info "Training models."
         model_dict = train_models(models, X, labels; parallelizer=exper.parallelizer, train_parallel=exper.train_parallel, cov=exper.coverage)
     else
-        # Pre-trained models:
-        if !(is_multi_processed(exper) && MPI.Comm_rank(exper.parallelizer.comm) != 0)
-            # Load models on root process:
-            @info "Loading pre-trained models."
-            model_dict = Serialization.deserialize(joinpath(pretrained_path(exper), "$(exper.save_name)_models.jls"))
-        else
-            # Dummy model on other processes: 
-            model_dict = nothing
-        end
-        # Broadcast models:
-        if is_multi_processed(exper)
-            model_dict = MPI.bcast(model_dict, exper.parallelizer.comm; root=0)
-            MPI.Barrier(exper.parallelizer.comm)
-        end
+        # Load models on root process:
+        @info "Loading pre-trained models."
+        model_dict = Serialization.deserialize(joinpath(pretrained_path(exper), "$(exper.save_name)_models.jls"))
     end
 
     # Save models: