From e7e91a3a5bffab7a8be2e7722f96e5ae91501c5d Mon Sep 17 00:00:00 2001
From: Pat Alt <55311242+pat-alt@users.noreply.github.com>
Date: Fri, 15 Sep 2023 16:45:47 +0200
Subject: [PATCH] added innovation for MNIST
---
.../generators/california_housing.sh | 2 +-
.../jobscripts/generators/credit_default.sh | 2 +-
.../jobscripts/generators/german_credit.sh | 2 +-
experiments/jobscripts/generators/gmsc.sh | 2 +-
.../jobscripts/generators/innovation/mnist.sh | 14 +++++
experiments/jobscripts/generators/mnist.sh | 2 +-
.../jobscripts/generators/synthetic.sh | 2 +-
experiments/jobscripts/generators/tabular.sh | 2 +-
.../jobscripts/tuning/generators/synthetic.sh | 2 +-
.../jobscripts/tuning/generators/tabular.sh | 2 +-
experiments/mnist.jl | 61 +++++++++++--------
experiments/models/default_models.jl | 2 +-
12 files changed, 58 insertions(+), 37 deletions(-)
create mode 100644 experiments/jobscripts/generators/innovation/mnist.sh
diff --git a/experiments/jobscripts/generators/california_housing.sh b/experiments/jobscripts/generators/california_housing.sh
index cd13e1c8..b0364af2 100644
--- a/experiments/jobscripts/generators/california_housing.sh
+++ b/experiments/jobscripts/generators/california_housing.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="California Housing (ECCCo)"
#SBATCH --time=3:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/credit_default.sh b/experiments/jobscripts/generators/credit_default.sh
index 3f46774b..93945bf1 100644
--- a/experiments/jobscripts/generators/credit_default.sh
+++ b/experiments/jobscripts/generators/credit_default.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="Credit Default (ECCCo)"
#SBATCH --time=3:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/german_credit.sh b/experiments/jobscripts/generators/german_credit.sh
index f07bdd23..07bc4df7 100644
--- a/experiments/jobscripts/generators/german_credit.sh
+++ b/experiments/jobscripts/generators/german_credit.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="German Credit (ECCCo)"
#SBATCH --time=3:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/gmsc.sh b/experiments/jobscripts/generators/gmsc.sh
index a9437ba3..d6c9223d 100644
--- a/experiments/jobscripts/generators/gmsc.sh
+++ b/experiments/jobscripts/generators/gmsc.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="GMSC (ECCCo)"
#SBATCH --time=3:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/innovation/mnist.sh b/experiments/jobscripts/generators/innovation/mnist.sh
new file mode 100644
index 00000000..8823002a
--- /dev/null
+++ b/experiments/jobscripts/generators/innovation/mnist.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#SBATCH --job-name="MNIST (ECCCo)"
+#SBATCH --time=24:00:00
+#SBATCH --ntasks=48
+#SBATCH --cpus-per-task=1
+#SBATCH --partition=compute
+#SBATCH --mem-per-cpu=4GB
+#SBATCH --account=innovation
+#SBATCH --mail-type=END # Set mail type to 'END' to receive a mail when the job finishes.
+
+module load 2023r1 openmpi
+
+srun julia --project=experiments experiments/run_experiments.jl -- data=mnist output_path=results mpi > experiments/mnist.log
diff --git a/experiments/jobscripts/generators/mnist.sh b/experiments/jobscripts/generators/mnist.sh
index 17301eaa..7f16991b 100644
--- a/experiments/jobscripts/generators/mnist.sh
+++ b/experiments/jobscripts/generators/mnist.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="MNIST (ECCCo)"
#SBATCH --time=24:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/synthetic.sh b/experiments/jobscripts/generators/synthetic.sh
index 4b47382d..15e4f310 100644
--- a/experiments/jobscripts/generators/synthetic.sh
+++ b/experiments/jobscripts/generators/synthetic.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="Synthetic (ECCCo)"
#SBATCH --time=02:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/generators/tabular.sh b/experiments/jobscripts/generators/tabular.sh
index dbd28c02..f7eae4ea 100644
--- a/experiments/jobscripts/generators/tabular.sh
+++ b/experiments/jobscripts/generators/tabular.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="Tabular (ECCCo)"
#SBATCH --time=05:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/tuning/generators/synthetic.sh b/experiments/jobscripts/tuning/generators/synthetic.sh
index 51d52da1..10246f04 100644
--- a/experiments/jobscripts/tuning/generators/synthetic.sh
+++ b/experiments/jobscripts/tuning/generators/synthetic.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="Grid-search Synthetic (ECCCo)"
#SBATCH --time=03:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/jobscripts/tuning/generators/tabular.sh b/experiments/jobscripts/tuning/generators/tabular.sh
index 556d97ac..1aa26ced 100644
--- a/experiments/jobscripts/tuning/generators/tabular.sh
+++ b/experiments/jobscripts/tuning/generators/tabular.sh
@@ -2,7 +2,7 @@
#SBATCH --job-name="Grid-search Tabular (ECCCo)"
#SBATCH --time=06:00:00
-#SBATCH --ntasks=100
+#SBATCH --ntasks=1000
#SBATCH --cpus-per-task=1
#SBATCH --partition=compute
#SBATCH --mem-per-cpu=4GB
diff --git a/experiments/mnist.jl b/experiments/mnist.jl
index a72e25db..0e509391 100644
--- a/experiments/mnist.jl
+++ b/experiments/mnist.jl
@@ -1,4 +1,5 @@
# Training data:
+dataname = "MNIST"
n_obs = 10000
counterfactual_data = load_mnist(n_obs)
counterfactual_data.X = ECCCo.pre_process.(counterfactual_data.X)
@@ -11,39 +12,45 @@ counterfactual_data.generative_model = vae
# Test data:
test_data = load_mnist_test()
+# Model tuning:
+model_tuning_params = DEFAULT_MODEL_TUNING_LARGE
+
+# Tuning parameters:
+tuning_params = DEFAULT_GENERATOR_TUNING
+
# Additional models:
add_models = Dict(
"LeNet-5" => lenet5,
- "ResNet-18" => resnet18(;epochs=10),
-)
-
-# Default builder:
-n_hidden = 128
-activation = Flux.swish
-builder = MLJFlux.@builder Flux.Chain(
- Dense(n_in, n_hidden, activation),
- Dense(n_hidden, n_out),
+ "ResNet-18" => resnet18(; epochs=10),
)
-# Number of individuals:
-n_ind = N_IND_SPECIFIED ? N_IND : 10
-
-# Run:
-run_experiment(
- counterfactual_data, test_data;
- dataname="MNIST",
- builder= builder,
- ð’Ÿx = Uniform(-1.0, 1.0),
- α = [1.0,1.0,1e-2],
- sampling_batch_size = 10,
+# Parameter choices:
+params = (
+ n_ind = N_IND_SPECIFIED ? N_IND : 10,
+ builder=default_builder(n_hidden=128, n_layers=2, activation=Flux.swish),
+ ð’Ÿx=Uniform(-1.0, 1.0),
+ α=[1.0, 1.0, 1e-2],
+ sampling_batch_size=10,
sampling_steps=50,
- use_ensembling = true,
- n_individuals = n_ind,
- use_variants = false,
- use_class_loss = true,
+ use_ensembling=true,
+ use_variants=false,
additional_models=add_models,
- epochs = 10,
- nsamples = 10,
+ epochs=10,
+ nsamples=10,
nmin=1,
niter_eccco=100
-)
\ No newline at end of file
+)
+
+if !GRID_SEARCH
+ run_experiment(
+ counterfactual_data, test_data;
+ dataname=dataname,
+ params...
+ )
+else
+ grid_search(
+ counterfactual_data, test_data;
+ dataname=dataname,
+ tuning_params=tuning_params
+ )
+end
\ No newline at end of file
diff --git a/experiments/models/default_models.jl b/experiments/models/default_models.jl
index ae009817..c50ff092 100644
--- a/experiments/models/default_models.jl
+++ b/experiments/models/default_models.jl
@@ -18,7 +18,7 @@ end
Default builder for MLPs.
"""
-function default_builder(n_hidden::Int=16, n_layers::Int=3, activation::Function=Flux.swish)
+function default_builder(;n_hidden::Int=16, n_layers::Int=3, activation::Function=Flux.swish)
builder = TuningBuilder(n_hidden=n_hidden, n_layers=n_layers, activation=activation)
return builder
end
--
GitLab