diff --git a/artifacts/results/images/eccco_illustration.png b/artifacts/results/images/eccco_illustration.png
new file mode 100644
index 0000000000000000000000000000000000000000..b50dbc5da2acdbe6f7df2647ce9313bc2906a42d
Binary files /dev/null and b/artifacts/results/images/eccco_illustration.png differ
diff --git a/artifacts/results/images/mnist_eccco.png b/artifacts/results/images/mnist_eccco.png
index c9cd7a83ff98b28026e4ed8b0ff09c5eceee55c6..c1a839162f51abfad2296abd0cafaa4d972503df 100644
Binary files a/artifacts/results/images/mnist_eccco.png and b/artifacts/results/images/mnist_eccco.png differ
diff --git a/artifacts/results/images/mnist_eccco_benchmark.png b/artifacts/results/images/mnist_eccco_benchmark.png
new file mode 100644
index 0000000000000000000000000000000000000000..9bf388b70297582d15f3accc2b6da83d46e512f1
Binary files /dev/null and b/artifacts/results/images/mnist_eccco_benchmark.png differ
diff --git a/artifacts/results/images/mnist_generated_JEM Ensemble.png b/artifacts/results/images/mnist_generated_JEM Ensemble.png
index a2d8a81bf78e3778ae2c9b68aae775f5639f74a7..f222cade2979638496bfc77d811b53dd850a4ed3 100644
Binary files a/artifacts/results/images/mnist_generated_JEM Ensemble.png and b/artifacts/results/images/mnist_generated_JEM Ensemble.png differ
diff --git a/artifacts/results/images/mnist_generated_JEM.png b/artifacts/results/images/mnist_generated_JEM.png
index db4e965990f722732a69f3b92f3c0bb2264cf43d..28a1647f4120deb6eafcdbe88dd653aa9033c90d 100644
Binary files a/artifacts/results/images/mnist_generated_JEM.png and b/artifacts/results/images/mnist_generated_JEM.png differ
diff --git a/artifacts/results/images/mnist_generated_MLP Ensemble.png b/artifacts/results/images/mnist_generated_MLP Ensemble.png
index 02468de5773695e05a723fdbe08541ed31a552b0..7a21fab7a509b8731c85f90db7da61427686afd0 100644
Binary files a/artifacts/results/images/mnist_generated_MLP Ensemble.png and b/artifacts/results/images/mnist_generated_MLP Ensemble.png differ
diff --git a/artifacts/results/images/mnist_generated_MLP.png b/artifacts/results/images/mnist_generated_MLP.png
index 029c632767e0e4b1d9ee14efcd7b2a7855322e88..970e9c6879f11d61a593a0028ac72dad421282c4 100644
Binary files a/artifacts/results/images/mnist_generated_MLP.png and b/artifacts/results/images/mnist_generated_MLP.png differ
diff --git a/artifacts/results/mnist_model_performance.csv b/artifacts/results/mnist_model_performance.csv
index 61dd7f8de0d3e2bc61fe9523b8b4717dc9e9b0ea..64433b21889e5308f8754b191f7e1c3241792341 100644
--- a/artifacts/results/mnist_model_performance.csv
+++ b/artifacts/results/mnist_model_performance.csv
@@ -1,5 +1,5 @@
acc,precision,f1score,mod_name
-0.9101,0.9104599093362784,0.9086219445938086,JEM Ensemble
-0.942,0.9415017061499791,0.9412674228675252,MLP
-0.9441,0.9435986059071019,0.9434079268844547,MLP Ensemble
-0.8752,0.8804593411643,0.8733208976945687,JEM
+0.9019,0.9013187021264487,0.9006731391569962,JEM Ensemble
+0.9431,0.9429692537123295,0.942515960085936,MLP
+0.9449,0.9443587962537836,0.9442779098066854,MLP Ensemble
+0.8374,0.8446918423076417,0.8342455855332602,JEM
diff --git a/artifacts/results/mnist_model_performance.jls b/artifacts/results/mnist_model_performance.jls
index f0a8759eb39e6ed0d858bd21fc462e4617e7ffcd..f18e2c3529f5cb8ffc839f5fb918032ef451b469 100644
Binary files a/artifacts/results/mnist_model_performance.jls and b/artifacts/results/mnist_model_performance.jls differ
diff --git a/artifacts/results/mnist_models.jls b/artifacts/results/mnist_models.jls
index df1acd84992a396709245a2a93d369f665f85622..bf6810b5397cc4a444ab90ae830a00c35c22d198 100644
Binary files a/artifacts/results/mnist_models.jls and b/artifacts/results/mnist_models.jls differ
diff --git a/artifacts/results/mnist_vae.jls b/artifacts/results/mnist_vae.jls
index dbbf8c47445a74c4a539ffda042084fc3873bdd6..1beb8495d7cc6c0d2a30194edf20c3e28b6625f4 100644
Binary files a/artifacts/results/mnist_vae.jls and b/artifacts/results/mnist_vae.jls differ
diff --git a/artifacts/results/mnist_vae_weak.jls b/artifacts/results/mnist_vae_weak.jls
index 2c434b6fb6e97be2e245a955fe2d594e718a9aeb..bb3131c46c4fd99db0cb695160b97bc4519e5d0f 100644
Binary files a/artifacts/results/mnist_vae_weak.jls and b/artifacts/results/mnist_vae_weak.jls differ
diff --git a/notebooks/mnist.qmd b/notebooks/mnist.qmd
index c630e681746e78c65dda4db8c1b05553401b2546..99eb283a4994af1fc6ffc3f02c29ca2f9c37904b 100644
--- a/notebooks/mnist.qmd
+++ b/notebooks/mnist.qmd
@@ -34,15 +34,15 @@ opt = Flux.Optimise.Adam(0.01)
```{julia}
# Search:
-generator = GenericGenerator(opt=opt)
+generic_generator = GenericGenerator(opt=opt)
ce_wachter = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
+ x_factual, target, counterfactual_data, M, generic_generator;
decision_threshold=γ, max_iter=T,
initialization=:identity,
)
-generator = GreedyGenerator(η=1.0)
+greedy_generator = GreedyGenerator(η=1.0)
ce_jsma = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
+ x_factual, target, counterfactual_data, M, greedy_generator;
decision_threshold=γ, max_iter=T,
initialization=:identity,
)
@@ -89,22 +89,22 @@ Serialization.serialize(joinpath(output_path,"mnist_vae_weak.jls"), vae_weak)
```{julia}
# Define generator:
-generator = REVISEGenerator(
+revise_generator = REVISEGenerator(
opt = opt,
λ=0.1
)
# Generate recourse:
counterfactual_data.generative_model = vae # assign generative model
ce_strong = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
+ x_factual, target, counterfactual_data, M, revise_generator;
decision_threshold=γ, max_iter=T,
initialization=:identity,
converge_when=:generator_conditions,
)
-counterfactual_data = deepcopy(counterfactual_data)
-counterfactual_data.generative_model = vae_weak
+counterfactual_data_weak = deepcopy(counterfactual_data)
+counterfactual_data_weak.generative_model = vae_weak
ce_weak = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
+ x_factual, target, counterfactual_data_weak, M, revise_generator;
decision_threshold=γ, max_iter=T,
initialization=:identity,
converge_when=:generator_conditions,
@@ -184,7 +184,7 @@ _finaliser = x -> x # finaliser function
sampler = ConditionalSampler(
ð’Ÿx, ð’Ÿy,
input_size=(input_dim,),
- batch_size=1,
+ batch_size=10,
)
α = [1.0,1.0,1e-2] # penalty strengths
```
@@ -317,6 +317,8 @@ CSV.write(joinpath(output_path, "mnist_model_performance.csv"), model_performanc
model_performance
```
+### Different Models
+
```{julia}
# ECCCo:
λ=[0.5,0.1,0.5]
@@ -324,7 +326,7 @@ temp=0.5
η=0.01
# Generate counterfactuals using ECCCo generator:
-generator = ECCCoGenerator(
+eccco_generator = ECCCoGenerator(
λ=λ,
temp=temp,
opt=Flux.Optimise.Adam(η),
@@ -333,7 +335,7 @@ generator = ECCCoGenerator(
ces = Dict()
for (mod_name, mod) in model_dict
ce = generate_counterfactual(
- x_factual, target, counterfactual_data, mod, generator;
+ x_factual, target, counterfactual_data, mod, eccco_generator;
decision_threshold=γ, max_iter=T,
initialization=:identity,
converge_when=:generator_conditions,
@@ -370,150 +372,31 @@ display(plt)
savefig(plt, joinpath(output_images_path, "mnist_eccco.png"))
```
-```{julia}
-Random.seed!(1234)
-
-# Set up search:
-factual_label = 9
-x_factual = reshape(counterfactual_data.X[:,rand(findall(predict_label(M, counterfactual_data).==factual_label))],input_dim,1)
-target = 7
-factual = predict_label(M, counterfactual_data, x_factual)[1]
-γ = 0.9
-T = 100
-
-η=1.0
-
-# Generate counterfactual using generic generator:
-generator = GenericGenerator(opt=Flux.Optimise.Adam(0.01),)
-ce_wachter = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
-
-generator = GreedyGenerator(η=η)
-ce_jsma = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
-
-# ECCCo:
-λ=[0.1,0.1,0.1]
-temp=0.1
-
-# Generate counterfactual using ECCCo generator:
-generator = ECCCoGenerator(
- λ=λ,
- temp=temp,
- opt=Flux.Optimise.Adam(0.01),
-)
-ce_conformal = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
+### Different Generators
-# Generate counterfactual using ECCCo generator:
-generator = ECCCoGenerator(
- λ=λ,
- temp=temp,
- opt=CounterfactualExplanations.Generators.JSMADescent(η=η),
-)
-ce_conformal_jsma = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
+```{julia}
+# Setup:
+model = model_dict["JEM Ensemble"]
-# Plot:
-p1 = Plots.plot(
- convert2image(MNIST, reshape(x_factual,28,28)),
- axis=nothing,
- size=(img_height, img_height),
- title="Factual"
+# Benchmark generators:
+generator_dict = Dict(
+ :wachter => generic_generator,
+ :revise => revise_generator,
+ :greedy => greedy_generator,
+ :eccco => eccco_generator,
)
-plts = [p1]
-ces = [ce_wachter, ce_conformal, ce_jsma, ce_conformal_jsma]
-_names = ["Wachter", "ECCCo", "JSMA", "ECCCo-JSMA"]
-for x in zip(ces, _names)
- ce, _name = (x[1],x[2])
- x_factual = CounterfactualExplanations.counterfactual(ce)
- _phat = target_probs(ce)
- _title = "$_name (p̂=$(round(_phat[1]; digits=3)))"
- plt = Plots.plot(
- convert2image(MNIST, reshape(x_factual,28,28)),
- axis=nothing,
- size=(img_height, img_height),
- title=_title
+ces = Dict()
+for (gen_name, gen) in generator_dict
+ ce = generate_counterfactual(
+ x_factual, target, counterfactual_data, model, gen;
+ decision_threshold=γ, max_iter=T,
+ initialization=:identity,
+ converge_when=:generator_conditions,
)
- plts = [plts..., plt]
+ ces[gen_name] = ce
end
-plt = Plots.plot(plts...; size=(img_height*length(plts),img_height), layout=(1,length(plts)))
-display(plt)
-savefig(plt, joinpath(www_path, "eccco_mnist.png"))
-```
-
-```{julia}
-# Random.seed!(1234)
-
-# Set up search:
-factual_label = 8
-x_factual = reshape(counterfactual_data.X[:,rand(findall(predict_label(M, counterfactual_data).==factual_label))],input_dim,1)
-target = 3
-factual = predict_label(M, counterfactual_data, x_factual)[1]
-γ = 0.5
-T = 100
-
-# Generate counterfactual using generic generator:
-generator = GenericGenerator(opt=Flux.Optimise.Adam(),)
-ce_wachter = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
-)
-
-generator = GreedyGenerator(η=1.0)
-ce_jsma = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
-)
-
-# ECCCo:
-λ=[0.0,1.0]
-temp=0.5
-
-# Generate counterfactual using CCE generator:
-generator = CCEGenerator(
- λ=λ,
- temp=temp,
- opt=Flux.Optimise.Adam(),
-)
-ce_conformal = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
-
-# Generate counterfactual using CCE generator:
-generator = CCEGenerator(
- λ=λ,
- temp=temp,
- opt=CounterfactualExplanations.Generators.JSMADescent(η=1.0),
-)
-ce_conformal_jsma = generate_counterfactual(
- x_factual, target, counterfactual_data, M, generator;
- decision_threshold=γ, max_iter=T,
- initialization=:identity,
- converge_when=:generator_conditions,
-)
+plt_order = sortperm(collect(keys(ces)))
# Plot:
p1 = Plots.plot(
@@ -522,49 +405,25 @@ p1 = Plots.plot(
size=(img_height, img_height),
title="Factual"
)
-plts = [p1]
-ces = [ce_wachter, ce_conformal, ce_jsma, ce_conformal_jsma]
-_names = ["Wachter", "CCE", "JSMA", "CCE-JSMA"]
-for x in zip(ces, _names)
- ce, _name = (x[1],x[2])
- x = CounterfactualExplanations.counterfactual(ce)
+plts = []
+for (_name,ce) in ces
+ _x = CounterfactualExplanations.counterfactual(ce)
_phat = target_probs(ce)
_title = "$_name (p̂=$(round(_phat[1]; digits=3)))"
plt = Plots.plot(
- convert2image(MNIST, reshape(x,28,28)),
+ convert2image(MNIST, reshape(_x,28,28)),
axis=nothing,
size=(img_height, img_height),
title=_title
)
plts = [plts..., plt]
end
+plts = plts[plt_order]
+plts = [p1, plts...]
plt = Plots.plot(plts...; size=(img_height*length(plts),img_height), layout=(1,length(plts)))
display(plt)
-savefig(plt, joinpath(www_path, "cce_mnist.png"))
-```
-
-```{julia}
-if M.model.model isa JointEnergyModels.JointEnergyClassifier
- jem = M.model.model.jem
- n_iter = 200
- _w = 1500
- plts = []
- neach = 10
- for i in 1:10
- x = jem.sampler(jem.chain, jem.sampling_rule; niter=n_iter, n_samples=neach, y=i)
- plts_i = []
- for j in 1:size(x, 2)
- xj = x[:,j]
- xj = reshape(xj, (n_digits, n_digits))
- plts_i = [plts_i..., Plots.heatmap(rotl90(xj), axis=nothing, cb=false)]
- end
- plt = Plots.plot(plts_i..., size=(_w,0.10*_w), layout=(1,10))
- plts = [plts..., plt]
- end
- plt = Plots.plot(plts..., size=(_w,_w), layout=(10,1))
- display(plt)
-end
+savefig(plt, joinpath(output_images_path, "mnist_eccco_benchmark.png"))
```
## Benchmark
diff --git a/paper/paper.pdf b/paper/paper.pdf
index 61de5f8d38ffe417566685f948b7c429c3cccd90..ca87169a5b0c6e9867f9fbce9a18262852f3c448 100644
Binary files a/paper/paper.pdf and b/paper/paper.pdf differ
diff --git a/paper/paper.tex b/paper/paper.tex
index 80c3ae33ff74a2fb3ee664c9238b35238532e2fc..32de8391f5f72c02f99e6e6f05f53fc15f133328 100644
--- a/paper/paper.tex
+++ b/paper/paper.tex
@@ -133,7 +133,7 @@ While Counterfactual Explanations can be generated for arbitrary regression mode
\end{aligned}
\end{equation}
-Here $\text{yloss}$ denotes the primary loss function already introduced above and $\text{cost}$ is either a single penalty or a collection of penalties that are used to impose constraints through regularization. Following the convention in \citet{altmeyer2023endogenous} we use $\mathbf{Z}^\prime=\{ \mathbf{z}_m\}_M$ to denote the vector $M$-dimensional array of counterfactual states. This is to explicitly account for the fact that we can generate multiple counterfactuals $M$, as with DiCE \citep{mothilal2020explaining}, and may choose to traverse a latent representation $\mathcal{Z}$ of the feature space $\mathcal{X}$, as we will discuss further below.
+Here $\text{yloss}$ denotes the primary loss function already introduced above and $\text{cost}$ is either a single penalty or a collection of penalties that are used to impose constraints through regularization. Following the convention in \citet{altmeyer2023endogenous} we use $\mathbf{Z}^\prime=\{ \mathbf{z}_m\}_M$ to denote the $M$-dimensional array of counterfactual states. This is to explicitly account for the fact that we can generate multiple counterfactuals $M$, as with DiCE \citep{mothilal2020explaining}, and may choose to traverse a latent encoding $\mathcal{Z}$ of the feature space $\mathcal{X}$ where we denote $f^{-1}: \mathcal{X} \mapsto \mathcal{Z}$. Encodings may involve simple feature transformations or more advanced techniques involving generative models, as we will discuss further below.
Solutions to Equation~\ref{eq:general} are considered valid as soon as the predicted label matches the target label. A stripped-down counterfactual explanation is therefore little different from an adversarial example. In Figure~\ref{fig:adv}, for example, we have the baseline approach proposed in \citet{wachter2017counterfactual} to MNIST data (centre panel). This approach solves Equation~\ref{eq:general} through gradient-descent in the feature space with a penalty for the distance between the factual $\mathbf{x}$ and the counterfactual $\mathbf{x}^{\prime}$. The underlying classifier $M_{\theta}$ is a simple Multi-Layer Perceptron (MLP) with good test accuracy. For the generated counterfactual $\mathbf{x}^{\prime}$ the model predicts the target label with high confidence (centre panel in Figure~\ref{fig:adv}). The explanation is valid by definition, even though it looks a lot like an Adversarial Example \citep{goodfellow2014explaining}. \citet{schut2021generating} make the connection between Adversarial Examples and Counterfactual Explanations explicit and propose using a Jacobian-Based Saliency Map Attack (JSMA) to solve Equation~\ref{eq:general}. They demonstrate that this approach yields realistic and sparse counterfactuals for Bayesian, adversarially robust classifiers. Applying their approach to our simple MNIST classifier does not yield a realistic counterfactual but this one, too, is valid (right panel in Figure~\ref{fig:adv}).
@@ -221,7 +221,7 @@ In order to generate counterfactuals that are associated with low predictive unc
Here, $\kappa \in \{0,1\}$ is a hyper-parameter and $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha)$ can be interpreted as the probability of label $\mathbf{y}$ being included in the prediction set.
-In order to compute this penalty for any Black Box Model we merely need to perform a single calibration pass through a holdout set $\mathcal{D}_{\text{cal}}$. Arguably, data is typically abundant and in most applications, practitioners tend to hold out a test data set anyway. Consequently, CP removes the restriction on the family of predictive models, at the small cost of reserving a subset of the available data for calibration. Details concerning our implementation of Conformal Prediction can be found in Appendix~\ref{app-cp}.
+In order to compute this penalty for any Black Box Model we merely need to perform a single calibration pass through a holdout set $\mathcal{D}_{\text{cal}}$. Arguably, data is typically abundant and in most applications, practitioners tend to hold out a test data set anyway. Consequently, CP removes the restriction on the family of predictive models, at the small cost of reserving a subset of the available data for calibration. This particular case of conformal prediction is referred to as Split Conformal Prediction (SCP) as it involves splitting the training data into a proper training dataset and a calibration dataset. Details concerning our implementation of Conformal Prediction can be found in Appendix~\ref{app-cp}.
\subsection{Energy-Constrained Conformal Counterfactuals (ECCCo)}
@@ -236,27 +236,29 @@ Our framework for generating ECCCos combines the ideas introduced in the previou
where $\hat{\mathbf{x}}_{\theta}$ denotes samples generated using SGLD (Equation~\ref{eq:sgld}) and $\text{dist}(\cdot)$ is a generic term for a distance metric. Our default choice for $\text{dist}(\cdot)$ is the Manhatten Distance since it enforces sparsity.
-The first two terms in Equation~\ref{eq:eccco} correspond to the counterfactual search objective defined in~\citet{wachter2017counterfactual} which merely penalises the distance of counterfactuals from their factual values. The additional two penalties in ECCCo ensure that counterfactuals conform with the model's generative property and lead to minimally uncertain predictions, respectively. The hyperparameters $\lambda_1, ..., \lambda_3$ can be used to balance the different objectives: for example, we may choose to incur larger deviations from the factual in favour of conformity with the model's generative property by choosing lower values of $\lambda_1$ and relatively higher values of $\lambda_2$.
+The first two terms in Equation~\ref{eq:eccco} correspond to the counterfactual search objective defined in~\citet{wachter2017counterfactual} which merely penalises the distance of counterfactuals from their factual values. The additional two penalties in ECCCo ensure that counterfactuals conform with the model's generative property and lead to minimally uncertain predictions, respectively. The hyperparameters $\lambda_1, ..., \lambda_3$ can be used to balance the different objectives: for example, we may choose to incur larger deviations from the factual in favour of conformity with the model's generative property by choosing lower values of $\lambda_1$ and relatively higher values of $\lambda_2$. Figure~\ref{fig:eccco} illustrates this balancing act for an example involving synthetic data: vector fields indicate the direction of gradients with respect to the different components our proposed objective function (Equation~\ref{eq:eccco}).
+
+The entire procedure for Generating ECCCos is described in Algorithm~\ref{alg:eccco}. For the sake of simplicity and without loss of generality, we limit our attention to generating a single counterfactual $\mathbf{x}^\prime=f(\mathbf{z}^\prime)$ where in contrast to Equation~\ref{eq:eccco} $\mathbf{z}^\prime$ denotes a $1$-dimensional array containing a single counterfactual state. That state is initialized by passing the factual $\mathbf{x}$ through the encoder $f^{-1}$ which in our case corresponds to a simple feature transformer, rather than the encoder part of VAE as in REVISE~\citep{joshi2019realistic}.
\medskip
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
-\begin{minipage}[c]{0.45\textwidth}
+\begin{minipage}[c]{0.40\textwidth}
\centering
- \includegraphics[width=\textwidth]{../artifacts/results/images/surrogate_gone_wrong.png}
- \captionof{figure}{Using surrogates can improve plausibility, but also increases vulnerability. Counterfactuals for turning an 8 (eight) into a 3 (three): original image (left); counterfactual produced using REVISE \citep{joshi2019realistic} with a well-specified surrogate (centre); and a counterfactual produced using REVISE \citep{joshi2019realistic} with a poorly specified surrogate (right).} \label{fig:vae}
+ \includegraphics[width=\textwidth]{../artifacts/results/images/eccco_illustration.png}
+ \captionof{figure}{Vector fields indicating the direction of gradients with respect to the different components of the ECCCo objective (Equation~\ref{eq:eccco}).} \label{fig:eccco}
\end{minipage}
\hfill
-\begin{minipage}[c]{0.45\textwidth}
+\begin{minipage}[c]{0.50\textwidth}
\captionof{algorithm}{Generating ECCCos (For more details, see Appendix~\ref{app:eccco})}\label{alg:eccco}
\begin{algorithmic}[1]
\Require $\mathbf{x}, \mathbf{y}^*, M_{\theta}, f, \Lambda, \alpha, \mathcal{D}, T, \eta, m, M$ \linebreak where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^*$
\Ensure $\mathbf{x}^\prime$
\State Initialize $\mathbf{z}^\prime \gets f^{-1}(\mathbf{x})$
- \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$
\State Generate buffer $\mathcal{B}$ of $M$ conditional samples $\hat{\mathbf{x}}_{\theta}|\mathbf{y}^*$ using SGLD (Equation~\ref{eq:sgld})
+ \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$
\State Initialize $t \gets 0$
\While{\textit{not converged} or $t < T$}
\State $\hat{\mathbf{x}}_{\theta, t} \gets \text{rand}(\mathcal{B},m)$