diff --git a/AAAI/anonymous-submission-latex-2024.pdf b/AAAI/anonymous-submission-latex-2024.pdf
deleted file mode 100644
index 48f4c0913c4003d9d73fc7be31984582d925c191..0000000000000000000000000000000000000000
Binary files a/AAAI/anonymous-submission-latex-2024.pdf and /dev/null differ
diff --git a/AAAI/contents/table-real-world.tex b/AAAI/contents/table-real-world.tex
deleted file mode 100644
index 5d67371f126861a32c680147036bde6d875da62f..0000000000000000000000000000000000000000
--- a/AAAI/contents/table-real-world.tex
+++ /dev/null
@@ -1,45 +0,0 @@
-\begin{table}
-
-\caption{Results for real-world datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-real-world} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{llcccc}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{MNIST} & \multicolumn{2}{c}{GMSC} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6}
-Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓\\
-\midrule
- & ECCCo & \textbf{19.28 ± 5.01}** & 314.76 ± 32.36*\hphantom{*} & \textbf{79.16 ± 11.67}** & 18.26 ± 4.92**\\
-
- & REVISE & 188.70 ± 26.18*\hphantom{*} & \textbf{255.26 ± 41.50}** & 186.40 ± 28.06\hphantom{*}\hphantom{*} & \textbf{5.34 ± 2.38}**\\
-
- & Schut & 211.62 ± 27.13\hphantom{*}\hphantom{*} & 290.56 ± 40.66*\hphantom{*} & 200.98 ± 28.49\hphantom{*}\hphantom{*} & 6.50 ± 2.01**\\
-
-\multirow{-4}{*}{\raggedright\arraybackslash JEM} & Wachter & 222.90 ± 26.56\hphantom{*}\hphantom{*} & 361.88 ± 39.74\hphantom{*}\hphantom{*} & 214.08 ± 45.35\hphantom{*}\hphantom{*} & 61.04 ± 2.58\hphantom{*}\hphantom{*}\\
-\cmidrule{1-6}
- & ECCCo & \textbf{15.99 ± 3.06}** & 294.72 ± 30.75** & \textbf{83.28 ± 13.26}** & 17.21 ± 4.46**\\
-
- & REVISE & 173.59 ± 20.65** & \textbf{246.32 ± 37.46}** & 194.24 ± 35.41\hphantom{*}\hphantom{*} & \textbf{4.95 ± 1.26}**\\
-
- & Schut & 204.36 ± 23.14\hphantom{*}\hphantom{*} & 290.64 ± 39.49*\hphantom{*} & 208.45 ± 34.60\hphantom{*}\hphantom{*} & 6.12 ± 1.91**\\
-
-\multirow{-4}{*}{\raggedright\arraybackslash JEM Ensemble} & Wachter & 217.67 ± 23.78\hphantom{*}\hphantom{*} & 363.23 ± 39.24\hphantom{*}\hphantom{*} & 186.19 ± 33.88\hphantom{*}\hphantom{*} & 60.70 ± 44.32\hphantom{*}\hphantom{*}\\
-\cmidrule{1-6}
- & ECCCo & \textbf{41.95 ± 6.50}** & 591.58 ± 36.24\hphantom{*}\hphantom{*} & \textbf{75.93 ± 14.27}** & 17.20 ± 3.15**\\
-
- & REVISE & 365.82 ± 15.35*\hphantom{*} & \textbf{249.49 ± 41.55}** & 196.75 ± 41.25\hphantom{*}\hphantom{*} & \textbf{4.84 ± 0.60}**\\
-
- & Schut & 379.66 ± 17.16\hphantom{*}\hphantom{*} & 290.07 ± 42.65*\hphantom{*} & 212.00 ± 41.15\hphantom{*}\hphantom{*} & 6.44 ± 1.34**\\
-
-\multirow{-4}{*}{\raggedright\arraybackslash MLP} & Wachter & 386.05 ± 16.60\hphantom{*}\hphantom{*} & 361.83 ± 42.18\hphantom{*}\hphantom{*} & 218.34 ± 53.26\hphantom{*}\hphantom{*} & 45.84 ± 39.39\hphantom{*}\hphantom{*}\\
-\cmidrule{1-6}
- & ECCCo & \textbf{31.43 ± 3.91}** & 490.88 ± 27.19\hphantom{*}\hphantom{*} & \textbf{73.86 ± 14.63}** & 17.92 ± 4.17**\\
-
- & REVISE & 337.74 ± 11.89*\hphantom{*} & \textbf{247.67 ± 38.36}** & 207.21 ± 43.20\hphantom{*}\hphantom{*} & \textbf{5.78 ± 2.10}**\\
-
- & Schut & 354.80 ± 13.05\hphantom{*}\hphantom{*} & 285.79 ± 41.33*\hphantom{*} & 205.36 ± 32.11\hphantom{*}\hphantom{*} & 7.00 ± 2.15**\\
-
-\multirow{-4}{*}{\raggedright\arraybackslash MLP Ensemble} & Wachter & 360.79 ± 14.39\hphantom{*}\hphantom{*} & 357.73 ± 42.55\hphantom{*}\hphantom{*} & 213.71 ± 54.17\hphantom{*}\hphantom{*} & 73.09 ± 64.50\hphantom{*}\hphantom{*}\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table-synthetic.tex b/AAAI/contents/table-synthetic.tex
deleted file mode 100644
index df1746fbf5c7717ecbfc890e2f0e7551ab9834d8..0000000000000000000000000000000000000000
--- a/AAAI/contents/table-synthetic.tex
+++ /dev/null
@@ -1,37 +0,0 @@
-\begin{table}
-
-\caption{Results for synthetic datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-synthetic} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{llcccccc}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{Linearly Separable} & \multicolumn{2}{c}{Moons} & \multicolumn{2}{c}{Circles} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓\\
-\midrule
- & ECCCo & \textbf{0.03 ± 0.06}** & \textbf{0.20 ± 0.08}** & \textbf{0.31 ± 0.30}*\hphantom{*} & \textbf{1.20 ± 0.15}** & 0.52 ± 0.36\hphantom{*}\hphantom{*} & 1.22 ± 0.46\hphantom{*}\hphantom{*}\\
-
- & ECCCo (no CP) & 0.03 ± 0.06** & 0.20 ± 0.08** & 0.37 ± 0.30*\hphantom{*} & 1.21 ± 0.17** & 0.54 ± 0.39\hphantom{*}\hphantom{*} & 1.21 ± 0.46\hphantom{*}\hphantom{*}\\
-
- & ECCCo (no EBM) & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 0.34 ± 0.19\hphantom{*}\hphantom{*} & 0.91 ± 0.32\hphantom{*}\hphantom{*} & 1.71 ± 0.25\hphantom{*}\hphantom{*} & 0.70 ± 0.33\hphantom{*}\hphantom{*} & 1.30 ± 0.37\hphantom{*}\hphantom{*}\\
-
- & REVISE & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.41 ± 0.01** & 0.78 ± 0.23\hphantom{*}\hphantom{*} & 1.57 ± 0.26\hphantom{*}\hphantom{*} & \textbf{0.48 ± 0.16}*\hphantom{*} & \textbf{0.95 ± 0.32}*\hphantom{*}\\
-
- & Schut & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.73 ± 0.17\hphantom{*}\hphantom{*} & 0.67 ± 0.27\hphantom{*}\hphantom{*} & 1.50 ± 0.22*\hphantom{*} & 0.54 ± 0.43\hphantom{*}\hphantom{*} & 1.28 ± 0.53\hphantom{*}\hphantom{*}\\
-
-\multirow{-6}{*}{\raggedright\arraybackslash JEM} & Wachter & 0.18 ± 0.10\hphantom{*}\hphantom{*} & 0.44 ± 0.17\hphantom{*}\hphantom{*} & 0.80 ± 0.27\hphantom{*}\hphantom{*} & 1.78 ± 0.24\hphantom{*}\hphantom{*} & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 1.33 ± 0.32\hphantom{*}\hphantom{*}\\
-\cmidrule{1-8}
- & ECCCo & \textbf{0.29 ± 0.05}** & 0.23 ± 0.06** & 0.80 ± 0.62\hphantom{*}\hphantom{*} & 1.69 ± 0.40\hphantom{*}\hphantom{*} & 0.65 ± 0.53\hphantom{*}\hphantom{*} & 1.17 ± 0.41\hphantom{*}\hphantom{*}\\
-
- & ECCCo (no CP) & 0.29 ± 0.05** & \textbf{0.23 ± 0.07}** & \textbf{0.79 ± 0.62}\hphantom{*}\hphantom{*} & 1.68 ± 0.42\hphantom{*}\hphantom{*} & \textbf{0.49 ± 0.35}\hphantom{*}\hphantom{*} & 1.19 ± 0.44\hphantom{*}\hphantom{*}\\
-
- & ECCCo (no EBM) & 0.46 ± 0.05\hphantom{*}\hphantom{*} & 0.28 ± 0.04** & 1.34 ± 0.47\hphantom{*}\hphantom{*} & 1.68 ± 0.47\hphantom{*}\hphantom{*} & 0.84 ± 0.51\hphantom{*}\hphantom{*} & 1.23 ± 0.31\hphantom{*}\hphantom{*}\\
-
- & REVISE & 0.56 ± 0.05\hphantom{*}\hphantom{*} & 0.41 ± 0.01\hphantom{*}\hphantom{*} & 1.45 ± 0.44\hphantom{*}\hphantom{*} & \textbf{1.64 ± 0.31}\hphantom{*}\hphantom{*} & 0.58 ± 0.52\hphantom{*}\hphantom{*} & \textbf{0.95 ± 0.32}\hphantom{*}\hphantom{*}\\
-
- & Schut & 0.43 ± 0.06*\hphantom{*} & 0.47 ± 0.36\hphantom{*}\hphantom{*} & 1.45 ± 0.55\hphantom{*}\hphantom{*} & 1.73 ± 0.48\hphantom{*}\hphantom{*} & 0.58 ± 0.37\hphantom{*}\hphantom{*} & 1.23 ± 0.43\hphantom{*}\hphantom{*}\\
-
-\multirow{-6}{*}{\raggedright\arraybackslash MLP} & Wachter & 0.51 ± 0.04\hphantom{*}\hphantom{*} & 0.40 ± 0.08\hphantom{*}\hphantom{*} & 1.32 ± 0.41\hphantom{*}\hphantom{*} & 1.69 ± 0.32\hphantom{*}\hphantom{*} & 0.83 ± 0.50\hphantom{*}\hphantom{*} & 1.24 ± 0.29\hphantom{*}\hphantom{*}\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table_all.tex b/AAAI/contents/table_all.tex
deleted file mode 100644
index 84a0d04c0153148e937e0490d3dacdd5951389d6..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_all.tex
+++ /dev/null
@@ -1,147 +0,0 @@
-\begin{table}
-
-\caption{All results for all datasets: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-full} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{ccccccccc}
-\toprule
-Model & Data & Generator & Cost ↓ & Unfaithfulness ↓ & Implausibility ↓ & Redundancy ↑ & Uncertainty ↓ & Validity ↑\\
-\midrule
- & & ECCCo & 0.74 ± 0.21\hphantom{*}\hphantom{*} & 0.52 ± 0.36\hphantom{*}\hphantom{*} & 1.22 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & ECCCo (no CP) & 0.72 ± 0.21\hphantom{*}\hphantom{*} & 0.54 ± 0.39\hphantom{*}\hphantom{*} & 1.21 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & ECCCo (no EBM) & 0.52 ± 0.15\hphantom{*}\hphantom{*} & 0.70 ± 0.33\hphantom{*}\hphantom{*} & 1.30 ± 0.37\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 0.97 ± 0.34\hphantom{*}\hphantom{*} & \textbf{0.48 ± 0.16}*\hphantom{*} & \textbf{0.95 ± 0.32}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.50 ± 0.51\hphantom{*}\hphantom{*}\\
-
- & & Schut & 1.06 ± 0.43\hphantom{*}\hphantom{*} & 0.54 ± 0.43\hphantom{*}\hphantom{*} & 1.28 ± 0.53\hphantom{*}\hphantom{*} & \textbf{0.26 ± 0.25}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & \textbf{0.44 ± 0.16}\hphantom{*}\hphantom{*} & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 1.33 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 0.67 ± 0.19\hphantom{*}\hphantom{*} & 0.65 ± 0.53\hphantom{*}\hphantom{*} & 1.17 ± 0.41\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.09 ± 0.19** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.71 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.49 ± 0.35}\hphantom{*}\hphantom{*} & 1.19 ± 0.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.05 ± 0.16** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.45 ± 0.11\hphantom{*}\hphantom{*} & 0.84 ± 0.51\hphantom{*}\hphantom{*} & 1.23 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.23*\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 0.96 ± 0.31\hphantom{*}\hphantom{*} & 0.58 ± 0.52\hphantom{*}\hphantom{*} & \textbf{0.95 ± 0.32}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 0.50 ± 0.51\hphantom{*}\hphantom{*}\\
-
- & & Schut & 0.57 ± 0.11\hphantom{*}\hphantom{*} & 0.58 ± 0.37\hphantom{*}\hphantom{*} & 1.23 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.43 ± 0.18}** & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Circles} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & \textbf{0.40 ± 0.09}\hphantom{*}\hphantom{*} & 0.83 ± 0.50\hphantom{*}\hphantom{*} & 1.24 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.53 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 17.45 ± 2.92** & \textbf{79.16 ± 11.67}** & 18.26 ± 4.92** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.10 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 3.43 ± 1.67** & 186.40 ± 28.06\hphantom{*}\hphantom{*} & \textbf{5.34 ± 2.38}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.27 ± 0.33}** & 200.98 ± 28.49\hphantom{*}\hphantom{*} & 6.50 ± 2.01** & \textbf{0.77 ± 0.07}** & 0.07 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM} & Wachter & 57.71 ± 0.47\hphantom{*}\hphantom{*} & 214.08 ± 45.35\hphantom{*}\hphantom{*} & 61.04 ± 2.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.07 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 17.43 ± 3.04** & \textbf{83.28 ± 13.26}** & 17.21 ± 4.46** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 2.94 ± 1.13** & 194.24 ± 35.41\hphantom{*}\hphantom{*} & \textbf{4.95 ± 1.26}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.51 ± 0.29\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.03 ± 0.20}** & 208.45 ± 34.60\hphantom{*}\hphantom{*} & 6.12 ± 1.91** & \textbf{0.85 ± 0.05}** & 0.09 ± 0.04\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 56.79 ± 44.68\hphantom{*}\hphantom{*} & 186.19 ± 33.88\hphantom{*}\hphantom{*} & 60.70 ± 44.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.07 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 17.05 ± 2.87** & \textbf{75.93 ± 14.27}** & 17.20 ± 3.15** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.19 ± 0.08\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 2.93 ± 1.24** & 196.75 ± 41.25\hphantom{*}\hphantom{*} & \textbf{4.84 ± 0.60}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.38 ± 0.18\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & Schut & \textbf{1.49 ± 0.87}** & 212.00 ± 41.15\hphantom{*}\hphantom{*} & 6.44 ± 1.34** & \textbf{0.77 ± 0.13}** & 0.12 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash MLP} & Wachter & 42.97 ± 39.50\hphantom{*}\hphantom{*} & 218.34 ± 53.26\hphantom{*}\hphantom{*} & 45.84 ± 39.39\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.06 ± 0.06}\hphantom{*}\hphantom{*} & 0.50 ± 0.51\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 16.63 ± 2.62** & \textbf{73.86 ± 14.63}** & 17.92 ± 4.17** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.23 ± 0.07\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 3.73 ± 2.36** & 207.21 ± 43.20\hphantom{*}\hphantom{*} & \textbf{5.78 ± 2.10}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.33 ± 0.19\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
- & & Schut & \textbf{1.20 ± 0.47}** & 205.36 ± 32.11\hphantom{*}\hphantom{*} & 7.00 ± 2.15** & \textbf{0.79 ± 0.09}** & 0.12 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
-
-\multirow[t]{-16}{*}{\centering\arraybackslash GMSC} & \multirow[t]{-4}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 69.30 ± 66.00\hphantom{*}\hphantom{*} & 213.71 ± 54.17\hphantom{*}\hphantom{*} & 73.09 ± 64.50\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.06 ± 0.06}\hphantom{*}\hphantom{*} & 0.50 ± 0.51\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 0.75 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.03 ± 0.06}** & \textbf{0.20 ± 0.08}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.75 ± 0.17\hphantom{*}\hphantom{*} & 0.03 ± 0.06** & 0.20 ± 0.08** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.70 ± 0.16\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 0.34 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & REVISE & \textbf{0.41 ± 0.15}\hphantom{*}\hphantom{*} & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.41 ± 0.01** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.36 ± 0.36\hphantom{*}\hphantom{*} & 0.50 ± 0.51\hphantom{*}\hphantom{*}\\
-
- & & Schut & 1.15 ± 0.35\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.73 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.25}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.50 ± 0.13\hphantom{*}\hphantom{*} & 0.18 ± 0.10\hphantom{*}\hphantom{*} & 0.44 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.29 ± 0.05}** & 0.23 ± 0.06** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.94 ± 0.16\hphantom{*}\hphantom{*} & 0.29 ± 0.05** & \textbf{0.23 ± 0.07}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.60 ± 0.15\hphantom{*}\hphantom{*} & 0.46 ± 0.05\hphantom{*}\hphantom{*} & 0.28 ± 0.04** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.02 ± 0.10** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & REVISE & \textbf{0.42 ± 0.14}\hphantom{*}\hphantom{*} & 0.56 ± 0.05\hphantom{*}\hphantom{*} & 0.41 ± 0.01\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.47 ± 0.50\hphantom{*}\hphantom{*} & 0.48 ± 0.50\hphantom{*}\hphantom{*}\\
-
- & & Schut & 0.77 ± 0.17\hphantom{*}\hphantom{*} & 0.43 ± 0.06*\hphantom{*} & 0.47 ± 0.36\hphantom{*}\hphantom{*} & \textbf{0.20 ± 0.25}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Linearly Separable} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.51 ± 0.15\hphantom{*}\hphantom{*} & 0.51 ± 0.04\hphantom{*}\hphantom{*} & 0.40 ± 0.08\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.59 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 334.61 ± 46.37\hphantom{*}\hphantom{*} & \textbf{19.28 ± 5.01}** & 314.76 ± 32.36*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 4.43 ± 0.56\hphantom{*}\hphantom{*} & \textbf{0.98 ± 0.12}\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 170.68 ± 63.26\hphantom{*}\hphantom{*} & 188.70 ± 26.18*\hphantom{*} & \textbf{255.26 ± 41.50}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 4.39 ± 0.91\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.44 ± 1.60}** & 211.00 ± 27.21\hphantom{*}\hphantom{*} & 286.61 ± 39.85*\hphantom{*} & \textbf{0.99 ± 0.00}** & \textbf{1.08 ± 1.95}*\hphantom{*} & 0.24 ± 0.43\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM} & Wachter & 128.36 ± 14.95\hphantom{*}\hphantom{*} & 222.90 ± 26.56\hphantom{*}\hphantom{*} & 361.88 ± 39.74\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 4.37 ± 0.98\hphantom{*}\hphantom{*} & 0.95 ± 0.21\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 342.64 ± 41.14\hphantom{*}\hphantom{*} & \textbf{15.99 ± 3.06}** & 294.72 ± 30.75** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.07 ± 0.06** & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 170.21 ± 58.02\hphantom{*}\hphantom{*} & 173.59 ± 20.65** & \textbf{246.32 ± 37.46}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.56 ± 0.83\hphantom{*}\hphantom{*} & 0.93 ± 0.26\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.78 ± 1.02}** & 205.33 ± 24.07\hphantom{*}\hphantom{*} & 287.39 ± 39.33*\hphantom{*} & \textbf{0.99 ± 0.00}** & \textbf{0.32 ± 0.94}** & 0.11 ± 0.31\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 135.07 ± 16.79\hphantom{*}\hphantom{*} & 217.67 ± 23.78\hphantom{*}\hphantom{*} & 363.23 ± 39.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.93 ± 0.77\hphantom{*}\hphantom{*} & 0.94 ± 0.23\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 605.17 ± 44.78\hphantom{*}\hphantom{*} & \textbf{41.95 ± 6.50}** & 591.58 ± 36.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.57 ± 0.00** & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 146.61 ± 36.96\hphantom{*}\hphantom{*} & 365.82 ± 15.35*\hphantom{*} & \textbf{249.49 ± 41.55}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.62 ± 0.30\hphantom{*}\hphantom{*} & 0.87 ± 0.34\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.95 ± 0.37}** & 382.44 ± 17.81\hphantom{*}\hphantom{*} & 285.98 ± 42.48*\hphantom{*} & \textbf{0.99 ± 0.00}** & \textbf{0.05 ± 0.19}** & 0.06 ± 0.24\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash MLP} & Wachter & 136.08 ± 16.09\hphantom{*}\hphantom{*} & 386.05 ± 16.60\hphantom{*}\hphantom{*} & 361.83 ± 42.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.68 ± 0.36\hphantom{*}\hphantom{*} & 0.84 ± 0.36\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 525.87 ± 34.00\hphantom{*}\hphantom{*} & \textbf{31.43 ± 3.91}** & 490.88 ± 27.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.29 ± 0.00** & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 146.60 ± 35.64\hphantom{*}\hphantom{*} & 337.74 ± 11.89*\hphantom{*} & \textbf{247.67 ± 38.36}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.39 ± 0.22\hphantom{*}\hphantom{*} & 0.85 ± 0.36\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.98 ± 0.25}** & 359.54 ± 14.52\hphantom{*}\hphantom{*} & 283.99 ± 41.08*\hphantom{*} & \textbf{0.99 ± 0.00}** & \textbf{0.03 ± 0.14}** & 0.06 ± 0.24\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-16}{*}{\centering\arraybackslash MNIST} & \multirow[t]{-4}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 137.53 ± 18.95\hphantom{*}\hphantom{*} & 360.79 ± 14.39\hphantom{*}\hphantom{*} & 357.73 ± 42.55\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.47 ± 0.64\hphantom{*}\hphantom{*} & 0.80 ± 0.40\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 1.56 ± 0.44\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.30}*\hphantom{*} & \textbf{1.20 ± 0.15}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}**\\
-
- & & ECCCo (no CP) & 1.56 ± 0.46\hphantom{*}\hphantom{*} & 0.37 ± 0.30*\hphantom{*} & 1.21 ± 0.17** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}**\\
-
- & & ECCCo (no EBM) & 0.80 ± 0.25\hphantom{*}\hphantom{*} & 0.91 ± 0.32\hphantom{*}\hphantom{*} & 1.71 ± 0.25\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}**\\
-
- & & REVISE & 1.04 ± 0.43\hphantom{*}\hphantom{*} & 0.78 ± 0.23\hphantom{*}\hphantom{*} & 1.57 ± 0.26\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{1.00 ± 0.00}**\\
-
- & & Schut & 1.12 ± 0.31\hphantom{*}\hphantom{*} & 0.67 ± 0.27\hphantom{*}\hphantom{*} & 1.50 ± 0.22*\hphantom{*} & \textbf{0.08 ± 0.19}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & \textbf{0.72 ± 0.24}\hphantom{*}\hphantom{*} & 0.80 ± 0.27\hphantom{*}\hphantom{*} & 1.78 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.02 ± 0.10\hphantom{*}\hphantom{*} & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 2.18 ± 1.05\hphantom{*}\hphantom{*} & 0.80 ± 0.62\hphantom{*}\hphantom{*} & 1.69 ± 0.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.24*\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 2.07 ± 1.15\hphantom{*}\hphantom{*} & \textbf{0.79 ± 0.62}\hphantom{*}\hphantom{*} & 1.68 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.24*\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 1.25 ± 0.92\hphantom{*}\hphantom{*} & 1.34 ± 0.47\hphantom{*}\hphantom{*} & 1.68 ± 0.47\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.43 ± 0.18\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 0.79 ± 0.19*\hphantom{*} & 1.45 ± 0.44\hphantom{*}\hphantom{*} & \textbf{1.64 ± 0.31}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.40 ± 0.22\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{0.73 ± 0.25}*\hphantom{*} & 1.45 ± 0.55\hphantom{*}\hphantom{*} & 1.73 ± 0.48\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.28}*\hphantom{*} & \textbf{0.00 ± 0.00}** & 0.90 ± 0.30\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Moons} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 1.08 ± 0.83\hphantom{*}\hphantom{*} & 1.32 ± 0.41\hphantom{*}\hphantom{*} & 1.69 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.52 ± 0.08\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table_all_valid.tex b/AAAI/contents/table_all_valid.tex
deleted file mode 100644
index 3fa029f190735829c3f86cb011d1b1d802f304d6..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_all_valid.tex
+++ /dev/null
@@ -1,147 +0,0 @@
-\begin{table}
-
-\caption{All results for all datasets: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-full-valid} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{ccccccccc}
-\toprule
-Model & Data & Generator & Cost ↓ & Unfaithfulness ↓ & Implausibility ↓ & Redundancy ↑ & Uncertainty ↓ & Validity ↑\\
-\midrule
- & & ECCCo & 0.74 ± 0.21\hphantom{*}\hphantom{*} & 0.52 ± 0.36\hphantom{*}\hphantom{*} & 1.22 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.72 ± 0.21\hphantom{*}\hphantom{*} & 0.54 ± 0.39\hphantom{*}\hphantom{*} & 1.21 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.52 ± 0.15\hphantom{*}\hphantom{*} & 0.70 ± 0.33\hphantom{*}\hphantom{*} & 1.30 ± 0.37\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 1.28 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.33 ± 0.01}** & \textbf{0.64 ± 0.00}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & 1.06 ± 0.43\hphantom{*}\hphantom{*} & 0.54 ± 0.43\hphantom{*}\hphantom{*} & 1.28 ± 0.53\hphantom{*}\hphantom{*} & \textbf{0.26 ± 0.25}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & \textbf{0.45 ± 0.15}\hphantom{*}\hphantom{*} & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 1.33 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 0.67 ± 0.19\hphantom{*}\hphantom{*} & 0.65 ± 0.53\hphantom{*}\hphantom{*} & 1.17 ± 0.41\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.09 ± 0.19** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.71 ± 0.16\hphantom{*}\hphantom{*} & 0.49 ± 0.35\hphantom{*}\hphantom{*} & 1.19 ± 0.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.05 ± 0.16** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.45 ± 0.11\hphantom{*}\hphantom{*} & 0.84 ± 0.51\hphantom{*}\hphantom{*} & 1.23 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.23*\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 1.24 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.06 ± 0.01}** & \textbf{0.64 ± 0.00}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & 0.57 ± 0.11\hphantom{*}\hphantom{*} & 0.58 ± 0.37\hphantom{*}\hphantom{*} & 1.23 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.43 ± 0.18}** & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Circles} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & \textbf{0.40 ± 0.09}\hphantom{*}\hphantom{*} & 0.83 ± 0.50\hphantom{*}\hphantom{*} & 1.24 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.53 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 17.45 ± 2.92** & \textbf{79.16 ± 11.67}** & 18.26 ± 4.92** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.10 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 3.43 ± 1.67** & 186.40 ± 28.06\hphantom{*}\hphantom{*} & \textbf{5.34 ± 2.38}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.27 ± 0.33}** & 200.98 ± 28.49\hphantom{*}\hphantom{*} & 6.50 ± 2.01** & \textbf{0.77 ± 0.07}** & 0.07 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM} & Wachter & 57.71 ± 0.47\hphantom{*}\hphantom{*} & 214.08 ± 45.35\hphantom{*}\hphantom{*} & 61.04 ± 2.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.07 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 17.43 ± 3.04** & \textbf{83.28 ± 13.26}** & 17.21 ± 4.46** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 2.94 ± 1.13** & 194.24 ± 35.41\hphantom{*}\hphantom{*} & \textbf{4.95 ± 1.26}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.51 ± 0.29\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.03 ± 0.20}** & 208.45 ± 34.60\hphantom{*}\hphantom{*} & 6.12 ± 1.91** & \textbf{0.85 ± 0.05}** & 0.09 ± 0.04\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 56.79 ± 44.68\hphantom{*}\hphantom{*} & 186.19 ± 33.88\hphantom{*}\hphantom{*} & 60.70 ± 44.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.07 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 17.05 ± 2.87\hphantom{*}\hphantom{*} & \textbf{75.93 ± 14.27}** & 17.20 ± 3.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.19 ± 0.08\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 2.93 ± 1.24*\hphantom{*} & 196.75 ± 41.25\hphantom{*}\hphantom{*} & \textbf{4.84 ± 0.60}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.38 ± 0.18\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.49 ± 0.87}** & 212.00 ± 41.15\hphantom{*}\hphantom{*} & 6.44 ± 1.34\hphantom{*}\hphantom{*} & \textbf{0.77 ± 0.13}** & 0.12 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash MLP} & Wachter & 4.48 ± 0.18\hphantom{*}\hphantom{*} & 184.03 ± 48.16\hphantom{*}\hphantom{*} & 7.49 ± 0.89\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.12 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 16.63 ± 2.62\hphantom{*}\hphantom{*} & \textbf{73.86 ± 14.63}** & 17.92 ± 4.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.23 ± 0.07\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 3.73 ± 2.36\hphantom{*}\hphantom{*} & 207.21 ± 43.20\hphantom{*}\hphantom{*} & \textbf{5.78 ± 2.10}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.33 ± 0.19\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{1.20 ± 0.47}** & 205.36 ± 32.11\hphantom{*}\hphantom{*} & 7.00 ± 2.15*\hphantom{*} & \textbf{0.79 ± 0.09}** & 0.12 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-16}{*}{\centering\arraybackslash GMSC} & \multirow[t]{-4}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 4.97 ± 0.47\hphantom{*}\hphantom{*} & 177.20 ± 25.86\hphantom{*}\hphantom{*} & 10.27 ± 3.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 0.75 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.03 ± 0.06}** & \textbf{0.20 ± 0.08}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.75 ± 0.17\hphantom{*}\hphantom{*} & 0.03 ± 0.06** & 0.20 ± 0.08** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.70 ± 0.16\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 0.34 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & \textbf{0.41 ± 0.14}\hphantom{*}\hphantom{*} & 0.15 ± 0.00** & 0.41 ± 0.01** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.72 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & 1.15 ± 0.35\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.73 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.25}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.50 ± 0.13\hphantom{*}\hphantom{*} & 0.18 ± 0.10\hphantom{*}\hphantom{*} & 0.44 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.29 ± 0.05}** & 0.23 ± 0.06** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 0.94 ± 0.16\hphantom{*}\hphantom{*} & 0.29 ± 0.05** & \textbf{0.23 ± 0.07}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.60 ± 0.15\hphantom{*}\hphantom{*} & 0.46 ± 0.05\hphantom{*}\hphantom{*} & 0.28 ± 0.04** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.02 ± 0.10** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & \textbf{0.39 ± 0.15}\hphantom{*}\hphantom{*} & 0.52 ± 0.04\hphantom{*}\hphantom{*} & 0.41 ± 0.01\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & 0.77 ± 0.17\hphantom{*}\hphantom{*} & 0.43 ± 0.06*\hphantom{*} & 0.47 ± 0.36\hphantom{*}\hphantom{*} & \textbf{0.20 ± 0.25}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Linearly Separable} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.51 ± 0.15\hphantom{*}\hphantom{*} & 0.51 ± 0.04\hphantom{*}\hphantom{*} & 0.40 ± 0.08\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.59 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 334.98 ± 46.54\hphantom{*}\hphantom{*} & \textbf{19.27 ± 5.02}** & 314.54 ± 32.54*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{4.50 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 170.06 ± 62.45\hphantom{*}\hphantom{*} & 188.54 ± 26.22*\hphantom{*} & \textbf{254.32 ± 41.55}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 4.57 ± 0.14\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{7.63 ± 2.55}** & 199.70 ± 28.43\hphantom{*}\hphantom{*} & 273.01 ± 39.60** & \textbf{0.99 ± 0.00}** & 4.56 ± 0.13\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM} & Wachter & 128.13 ± 14.81\hphantom{*}\hphantom{*} & 222.81 ± 26.22\hphantom{*}\hphantom{*} & 361.38 ± 39.55\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 4.58 ± 0.16\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 342.64 ± 41.14\hphantom{*}\hphantom{*} & \textbf{15.99 ± 3.06}** & 294.72 ± 30.75** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{2.07 ± 0.06}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 171.95 ± 58.81\hphantom{*}\hphantom{*} & 173.05 ± 20.38** & \textbf{246.20 ± 37.74}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.76 ± 0.45\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{7.96 ± 2.49}** & 186.91 ± 22.98*\hphantom{*} & 264.68 ± 37.58** & \textbf{0.99 ± 0.00}** & 3.02 ± 0.26\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 134.98 ± 16.95\hphantom{*}\hphantom{*} & 217.37 ± 23.93\hphantom{*}\hphantom{*} & 362.91 ± 39.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 3.10 ± 0.31\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 605.17 ± 44.78\hphantom{*}\hphantom{*} & \textbf{41.95 ± 6.50}** & 591.58 ± 36.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.57 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 146.76 ± 37.07\hphantom{*}\hphantom{*} & 365.69 ± 14.90*\hphantom{*} & 245.36 ± 39.69** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.72 ± 0.18\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.25 ± 1.31}** & 371.12 ± 19.99\hphantom{*}\hphantom{*} & \textbf{245.11 ± 35.72}** & \textbf{0.99 ± 0.00}** & 0.75 ± 0.23\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-4}{*}{\centering\arraybackslash MLP} & Wachter & 135.08 ± 15.68\hphantom{*}\hphantom{*} & 384.76 ± 16.52\hphantom{*}\hphantom{*} & 359.21 ± 42.03\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.81 ± 0.22\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 525.87 ± 34.00\hphantom{*}\hphantom{*} & \textbf{31.43 ± 3.91}** & 490.88 ± 27.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.29 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 146.38 ± 35.18\hphantom{*}\hphantom{*} & 337.21 ± 11.68*\hphantom{*} & \textbf{244.84 ± 37.17}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.45 ± 0.16\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{9.75 ± 1.00}** & 344.60 ± 13.64*\hphantom{*} & 252.53 ± 37.92** & \textbf{0.99 ± 0.00}** & 0.55 ± 0.21\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-16}{*}{\centering\arraybackslash MNIST} & \multirow[t]{-4}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 134.48 ± 17.69\hphantom{*}\hphantom{*} & 358.51 ± 13.18\hphantom{*}\hphantom{*} & 352.63 ± 39.93\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.58 ± 0.67\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{1-9}
- & & ECCCo & 1.56 ± 0.44\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.30}*\hphantom{*} & \textbf{1.20 ± 0.15}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 1.56 ± 0.46\hphantom{*}\hphantom{*} & 0.37 ± 0.30*\hphantom{*} & 1.21 ± 0.17** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 0.80 ± 0.25\hphantom{*}\hphantom{*} & 0.91 ± 0.32\hphantom{*}\hphantom{*} & 1.71 ± 0.25\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 1.04 ± 0.43\hphantom{*}\hphantom{*} & 0.78 ± 0.23\hphantom{*}\hphantom{*} & 1.57 ± 0.26\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & 1.13 ± 0.29\hphantom{*}\hphantom{*} & 0.66 ± 0.25\hphantom{*}\hphantom{*} & 1.47 ± 0.10** & \textbf{0.07 ± 0.18}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & \multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & \textbf{0.73 ± 0.24}\hphantom{*}\hphantom{*} & 0.78 ± 0.23\hphantom{*}\hphantom{*} & 1.75 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.02 ± 0.11\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\cmidrule{2-9}
- & & ECCCo & 2.18 ± 1.05\hphantom{*}\hphantom{*} & 0.80 ± 0.62\hphantom{*}\hphantom{*} & 1.69 ± 0.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.24*\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no CP) & 2.07 ± 1.15\hphantom{*}\hphantom{*} & \textbf{0.79 ± 0.62}\hphantom{*}\hphantom{*} & 1.68 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.15 ± 0.24*\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & ECCCo (no EBM) & 1.25 ± 0.92\hphantom{*}\hphantom{*} & 1.34 ± 0.47\hphantom{*}\hphantom{*} & 1.68 ± 0.47\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.43 ± 0.18\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & REVISE & 0.79 ± 0.19*\hphantom{*} & 1.45 ± 0.44\hphantom{*}\hphantom{*} & 1.64 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.40 ± 0.22\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
- & & Schut & \textbf{0.78 ± 0.17}*\hphantom{*} & 1.39 ± 0.50\hphantom{*}\hphantom{*} & \textbf{1.59 ± 0.26}\hphantom{*}\hphantom{*} & \textbf{0.28 ± 0.25}*\hphantom{*} & \textbf{0.00 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-
-\multirow[t]{-12}{*}{\centering\arraybackslash Moons} & \multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 1.08 ± 0.83\hphantom{*}\hphantom{*} & 1.32 ± 0.41\hphantom{*}\hphantom{*} & 1.69 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.52 ± 0.08\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table_ebm_params.tex b/AAAI/contents/table_ebm_params.tex
deleted file mode 100644
index e7fe1e0907384a1c0cf18ccdcef83720d6b91d45..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_ebm_params.tex
+++ /dev/null
@@ -1,17 +0,0 @@
-\begin{table}
-
-\caption{EBM hyperparemeter choices for our experiments. \label{tab:ebmparams} \newline}
-\centering
-\fontsize{8}{10}\selectfont
-\begin{tabular}[t]{rrrr}
-\toprule
-Dataset & SGLD Steps & Batch Size & $\lambda$\\
-\midrule
-Linearly Separable & 30 & 50 & 0.10\\
-Moons & 30 & 10 & 0.10\\
-Circles & 20 & 100 & 0.01\\
-MNIST & 25 & 10 & 0.01\\
-GMSC & 30 & 10 & 0.10\\
-\bottomrule
-\end{tabular}
-\end{table}
diff --git a/AAAI/contents/table_gen_params.tex b/AAAI/contents/table_gen_params.tex
deleted file mode 100644
index 84b89401bdaebddd0a0c92778fe91d5cc0122d2b..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_gen_params.tex
+++ /dev/null
@@ -1,17 +0,0 @@
-\begin{table}
-
-\caption{Generator hyperparameters. \label{tab:genparams} \newline}
-\centering
-\fontsize{8}{10}\selectfont
-\begin{tabular}[t]{rrrrr}
-\toprule
-Dataset & $\eta$ & $\lambda_1$ & $\lambda_2$ & $\lambda_3$\\
-\midrule
-Linearly Separable & 0.01 & 0.25 & 0.75 & 0.75\\
-Moons & 0.05 & 0.25 & 0.75 & 0.75\\
-Circles & 0.01 & 0.25 & 0.75 & 0.75\\
-MNIST & 0.10 & 0.10 & 0.25 & 0.25\\
-GMSC & 0.05 & 0.10 & 0.50 & 0.50\\
-\bottomrule
-\end{tabular}
-\end{table}
diff --git a/AAAI/contents/table_params.tex b/AAAI/contents/table_params.tex
deleted file mode 100644
index d0ccc5a3ef1f60a4ff3f5d39f629e41ae4d3d46e..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_params.tex
+++ /dev/null
@@ -1,19 +0,0 @@
-\begin{table}
-
-\caption{Paremeter choices for our experiments. \label{tab:params} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{rrrrrrrr}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{4}{c}{Network Architecture} & \multicolumn{2}{c}{Training} \\
-\cmidrule(l{3pt}r{3pt}){3-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Dataset & Sample Size & Hidden Units & Hidden Layers & Activation & Ensemble Size & Epochs & Batch Size\\
-\midrule
-Linearly Separable & 1000 & 16 & 3 & swish & 5 & 100 & 100\\
-Moons & 2500 & 32 & 3 & relu & 5 & 500 & 128\\
-Circles & 1000 & 32 & 3 & swish & 5 & 100 & 100\\
-MNIST & 10000 & 128 & 1 & swish & 5 & 100 & 128\\
-GMSC & 13370 & 128 & 2 & swish & 5 & 100 & 250\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table_perf.tex b/AAAI/contents/table_perf.tex
deleted file mode 100644
index 737a98e599580f7d0a38c2c1ec377b9fbed577a8..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_perf.tex
+++ /dev/null
@@ -1,41 +0,0 @@
-\begin{table}
-
-\caption{Various standard performance metrics for our different models grouped by dataset. \label{tab:perf} \newline}
-\centering
-\fontsize{8}{10}\selectfont
-\begin{tabular}[t]{rrrrr}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{3}{c}{Performance Metrics} \\
-\cmidrule(l{3pt}r{3pt}){3-5}
-Dataset & Model & Accuracy & Precision & F1-Score\\
-\midrule
- & JEM & 0.99 & 0.99 & 0.99\\
-
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Linearly Separable} & MLP & 0.99 & 0.99 & 0.99\\
-\cmidrule{1-5}
- & JEM & 1.00 & 1.00 & 1.00\\
-
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Moons} & MLP & 1.00 & 1.00 & 1.00\\
-\cmidrule{1-5}
- & JEM & 0.98 & 0.98 & 0.98\\
-
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Circles} & MLP & 1.00 & 1.00 & 1.00\\
-\cmidrule{1-5}
- & JEM & 0.83 & 0.84 & 0.83\\
-
- & JEM Ensemble & 0.90 & 0.90 & 0.89\\
-
- & MLP & 0.95 & 0.95 & 0.95\\
-
-\multirow[t]{-4}{*}{\raggedleft\arraybackslash MNIST} & MLP Ensemble & 0.95 & 0.95 & 0.95\\
-\cmidrule{1-5}
- & JEM & 0.73 & 0.75 & 0.73\\
-
- & JEM Ensemble & 0.73 & 0.75 & 0.73\\
-
- & MLP & 0.75 & 0.75 & 0.75\\
-
-\multirow[t]{-4}{*}{\raggedleft\arraybackslash GMSC} & MLP Ensemble & 0.75 & 0.75 & 0.75\\
-\bottomrule
-\end{tabular}
-\end{table}
diff --git a/AAAI/contents/table_real.tex b/AAAI/contents/table_real.tex
deleted file mode 100644
index 43365ededec415bb645a1224b72ed5a590a90529..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_real.tex
+++ /dev/null
@@ -1,45 +0,0 @@
-\begin{table}
-
-\caption{Results for real-world datasets. Standard deviations across samples are shown in parentheses. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-real} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{>{}cccccccc}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{California Housing} & \multicolumn{2}{c}{GMSC} & \multicolumn{2}{c}{MNIST} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Model & Generator & Non-conformity ↓ & Implausibility ↓ & Non-conformity ↓ & Implausibility ↓ & Non-conformity ↓ & Implausibility ↓\\
-\midrule
- & ECCCo & \textbf{236.79 (51.16)} & 39.78 (3.18) & \textbf{41.65 (17.24)**} & 40.57 (8.74)** & \textbf{116.09 (30.70)**} & 281.33 (41.51)**\\
-
- & REVISE & 284.51 (52.74) & \textbf{5.58 (0.81)**} & 74.89 (15.82)** & \textbf{6.01 (5.75)**} & 348.74 (65.65)** & \textbf{246.69 (36.69)**}\\
-
- & Schut & 263.55 (60.56) & 8.00 (2.03) & 76.23 (15.54)** & 6.02 (0.72)** & 355.58 (64.84)** & 270.06 (40.41)**\\
-
-\multirow{-4}{*}{\centering\arraybackslash \textbf{JEM}} & Wachter & 274.55 (51.17) & 7.32 (1.80) & 146.02 (64.48) & 128.93 (74.00) & 694.08 (50.86) & 630.99 (33.01)\\
-\cmidrule{1-8}
- & ECCCo & \textbf{249.44 (58.53)} & 35.09 (5.56) & \textbf{26.55 (12.94)**} & 33.65 (8.33)** & \textbf{89.89 (27.26)**} & 240.59 (37.41)**\\
-
- & REVISE & 268.45 (66.87) & \textbf{5.44 (0.74)**} & 52.47 (14.12)** & 6.69 (3.37)** & 292.52 (53.13)** & \textbf{240.50 (35.73)**}\\
-
- & Schut & 279.38 (63.23) & 7.64 (1.47) & 56.34 (15.00)** & \textbf{6.27 (1.06)**} & 319.45 (59.02)** & 266.80 (40.46)**\\
-
-\multirow{-4}{*}{\centering\arraybackslash \textbf{JEM Ensemble}} & Wachter & 268.59 (68.66) & 7.16 (1.46) & 125.72 (70.80) & 126.55 (93.75) & 582.52 (58.46) & 543.90 (44.24)\\
-\cmidrule{1-8}
- & ECCCo & \textbf{230.92 (48.86)} & 37.53 (5.40) & \textbf{46.90 (15.80)**} & 37.78 (8.40)** & \textbf{212.45 (36.70)**} & 649.63 (58.80)\\
-
- & REVISE & 281.10 (53.01) & \textbf{5.34 (0.67)**} & 81.08 (19.53)** & \textbf{4.60 (0.72)**} & 839.79 (77.14)* & \textbf{244.33 (38.69)**}\\
-
- & Schut & 285.12 (56.00) & 6.48 (1.18)** & 90.67 (20.80)** & 5.56 (0.81)** & 842.80 (82.01)* & 264.94 (42.18)**\\
-
-\multirow{-4}{*}{\centering\arraybackslash \textbf{MLP}} & Wachter & 262.50 (56.87) & 9.21 (10.41) & 191.68 (30.86) & 200.23 (15.05) & 982.32 (61.81) & 561.23 (45.08)\\
-\cmidrule{1-8}
- & ECCCo & \textbf{212.47 (59.27)*} & 38.17 (6.18) & \textbf{74.65 (144.69)*} & 71.87 (145.19) & \textbf{162.21 (36.21)**} & 587.65 (95.01)\\
-
- & REVISE & 284.65 (49.52) & \textbf{5.64 (1.13)*} & 80.90 (14.59)** & \textbf{5.20 (1.52)**} & 741.30 (125.98)* & \textbf{242.76 (41.16)**}\\
-
- & Schut & 269.19 (46.08) & 7.30 (1.94) & 85.63 (19.15)** & 6.00 (0.99)** & 754.35 (132.26) & 266.94 (42.55)**\\
-
-\multirow{-4}{*}{\centering\arraybackslash \textbf{MLP Ensemble}} & Wachter & 278.09 (73.65) & 7.32 (1.75) & 220.05 (17.41) & 203.65 (14.77) & 871.09 (92.36) & 536.24 (48.73)\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/contents/table_synth.tex b/AAAI/contents/table_synth.tex
deleted file mode 100644
index 9e966e12acb047725654086204112483da501ff6..0000000000000000000000000000000000000000
--- a/AAAI/contents/table_synth.tex
+++ /dev/null
@@ -1,37 +0,0 @@
-\begin{table}
-
-\caption{Results for synthetic datasets. Standard deviations across samples are shown in parentheses. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-synth} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{>{}cccccccc}
-\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{Circles} & \multicolumn{2}{c}{Linearly Separable} & \multicolumn{2}{c}{Moons} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Model & Generator & Non-conformity ↓ & Implausibility ↓ & Non-conformity ↓ & Implausibility ↓ & Non-conformity ↓ & Implausibility ↓\\
-\midrule
- & ECCCo & \textbf{0.63 (1.58)} & 1.44 (1.37) & 0.10 (0.06)** & 0.19 (0.03)** & \textbf{0.57 (0.58)**} & \textbf{1.29 (0.21)*}\\
-
- & ECCCo (no CP) & 0.64 (1.61) & 1.45 (1.38) & \textbf{0.10 (0.07)**} & \textbf{0.19 (0.03)**} & 0.63 (0.64)* & 1.30 (0.21)*\\
-
- & ECCCo (no EBM) & 1.41 (1.51) & 1.50 (1.38) & 0.37 (0.28) & 0.38 (0.26) & 1.73 (1.34) & 1.73 (1.42)\\
-
- & REVISE & 0.96 (0.32)* & \textbf{0.95 (0.32)*} & 0.41 (0.02)** & 0.41 (0.01)** & 1.59 (0.55) & 1.55 (0.20)\\
-
- & Schut & 0.99 (0.80) & 1.28 (0.53) & 0.66 (0.23) & 0.66 (0.22) & 1.55 (0.61) & 1.42 (0.16)*\\
-
-\multirow{-6}{*}{\centering\arraybackslash \textbf{JEM}} & Wachter & 1.41 (1.50) & 1.51 (1.35) & 0.44 (0.16) & 0.44 (0.15) & 1.77 (0.48) & 1.67 (0.15)\\
-\cmidrule{1-8}
- & ECCCo & \textbf{0.37 (0.65)**} & 1.30 (0.68) & \textbf{0.03 (0.02)**} & 0.69 (0.10) & 1.68 (1.74) & 2.02 (0.86)\\
-
- & ECCCo (no CP) & 0.50 (0.85)* & 1.28 (0.66) & \textbf{0.03 (0.02)**} & 0.68 (0.10) & \textbf{1.34 (1.66)} & 2.11 (0.88)\\
-
- & ECCCo (no EBM) & 2.00 (1.46) & 1.83 (1.00) & 1.25 (0.87) & 1.84 (1.10) & 2.98 (1.89) & 2.29 (1.75)\\
-
- & REVISE & 1.16 (1.05) & \textbf{0.95 (0.32)*} & 1.10 (0.10) & \textbf{0.40 (0.01)**} & 2.46 (1.05) & \textbf{1.54 (0.27)*}\\
-
- & Schut & 1.60 (1.15) & 1.24 (0.44) & 0.81 (0.10)* & 0.47 (0.24) & 2.71 (1.15) & 1.62 (0.42)\\
-
-\multirow{-6}{*}{\centering\arraybackslash \textbf{MLP}} & Wachter & 1.67 (1.05) & 1.31 (0.43) & 0.94 (0.11) & 0.44 (0.15) & 2.95 (1.42) & 1.84 (1.33)\\
-\bottomrule
-\end{tabular}}
-\end{table}
diff --git a/AAAI/paper.pdf b/AAAI/paper.pdf
deleted file mode 100644
index 543d9f3d33bd8c78089f64483de8d6fcba772436..0000000000000000000000000000000000000000
Binary files a/AAAI/paper.pdf and /dev/null differ
diff --git a/AAAI/paper.tex b/AAAI/paper.tex
deleted file mode 100644
index bd145d5edd7433a0af9690bc6471172e6b43f6b9..0000000000000000000000000000000000000000
--- a/AAAI/paper.tex
+++ /dev/null
@@ -1,430 +0,0 @@
-%File: anonymous-submission-latex-2024.tex
-\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
-\usepackage[submission]{aaai24} % DO NOT CHANGE THIS
-\usepackage{times} % DO NOT CHANGE THIS
-\usepackage{helvet} % DO NOT CHANGE THIS
-\usepackage{courier} % DO NOT CHANGE THIS
-\usepackage[hyphens]{url} % DO NOT CHANGE THIS
-\usepackage{graphicx} % DO NOT CHANGE THIS
-\urlstyle{rm} % DO NOT CHANGE THIS
-\def\UrlFont{\rm} % DO NOT CHANGE THIS
-\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
-\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
-\frenchspacing % DO NOT CHANGE THIS
-\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
-\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
-%
-% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
-\usepackage{algorithm}
-% \usepackage{algorithmic}
-
-%
-% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
-% \usepackage{newfloat}
-% \usepackage{listings}
-% \DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
-% \lstset{%
-% basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
-% numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
-% aboveskip=0pt,belowskip=0pt,%
-% showstringspaces=false,tabsize=2,breaklines=true}
-% \floatstyle{ruled}
-% \newfloat{listing}{tb}{lst}{}
-% \floatname{listing}{Listing}
-%
-% Keep the \pdfinfo as shown here. There's no need
-% for you to add the /Title and /Author tags.
-\pdfinfo{
-/TemplateVersion (2024.1)
-}
-
-\usepackage{amsfonts} % blackboard math symbols
-\usepackage{amsmath}
-\usepackage{amsthm}
-\usepackage{caption}
-\usepackage{graphicx}
-\usepackage{algpseudocode}
-\usepackage{import}
-\usepackage{booktabs}
-\usepackage{longtable}
-\usepackage{array}
-\usepackage{multirow}
-
-
-% Numbered Environments:
-\newtheorem{definition}{Definition}[section]
-\newtheorem{question}{Research Question}[section]
-
-% Bibliography
-% \bibliographystyle{unsrtnat}
-% \setcitestyle{numbers,square,comma}
-
-% Algorithm
-\renewcommand{\algorithmicrequire}{\textbf{Input:}}
-\renewcommand{\algorithmicensure}{\textbf{Output:}}
-
-% DISALLOWED PACKAGES
-% \usepackage{authblk} -- This package is specifically forbidden
-% \usepackage{balance} -- This package is specifically forbidden
-% \usepackage{color (if used in text)
-% \usepackage{CJK} -- This package is specifically forbidden
-% \usepackage{float} -- This package is specifically forbidden
-% \usepackage{flushend} -- This package is specifically forbidden
-% \usepackage{fontenc} -- This package is specifically forbidden
-% \usepackage{fullpage} -- This package is specifically forbidden
-% \usepackage{geometry} -- This package is specifically forbidden
-% \usepackage{grffile} -- This package is specifically forbidden
-% \usepackage{hyperref} -- This package is specifically forbidden
-% \usepackage{navigator} -- This package is specifically forbidden
-% (or any other package that embeds links such as navigator or hyperref)
-% \indentfirst} -- This package is specifically forbidden
-% \layout} -- This package is specifically forbidden
-% \multicol} -- This package is specifically forbidden
-% \nameref} -- This package is specifically forbidden
-% \usepackage{savetrees} -- This package is specifically forbidden
-% \usepackage{setspace} -- This package is specifically forbidden
-% \usepackage{stfloats} -- This package is specifically forbidden
-% \usepackage{tabu} -- This package is specifically forbidden
-% \usepackage{titlesec} -- This package is specifically forbidden
-% \usepackage{tocbibind} -- This package is specifically forbidden
-% \usepackage{ulem} -- This package is specifically forbidden
-% \usepackage{wrapfig} -- This package is specifically forbidden
-% DISALLOWED COMMANDS
-% \nocopyright -- Your paper will not be published if you use this command
-% \addtolength -- This command may not be used
-% \balance -- This command may not be used
-% \baselinestretch -- Your paper will not be published if you use this command
-% \clearpage -- No page breaks of any kind may be used for the final version of your paper
-% \columnsep -- This command may not be used
-% \newpage -- No page breaks of any kind may be used for the final version of your paper
-% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
-% \pagestyle -- This command may not be used
-% \tiny -- This is not an acceptable font size.
-% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
-% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
-
-\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
-
-% The file aaai24.sty is the style file for AAAI Press
-% proceedings, working notes, and technical reports.
-%
-
-% Title
-
-% Your title must be in mixed case, not sentence case.
-% That means all verbs (including short verbs like be, is, using,and go),
-% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
-% articles, conjunctions, and prepositions are lower case unless they
-% directly follow a colon or long dash
-\title{AAAI Press Anonymous Submission\\Instructions for Authors Using \LaTeX{}}
-\author{
- %Authors
- % All authors must be in the same font size and format.
- Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
- AAAI Style Contributions by Pater Patel Schneider,
- Sunil Issar,\\
- J. Scott Penberthy,
- George Ferguson,
- Hans Guesgen,
- Francisco Cruz\equalcontrib,
- Marc Pujol-Gonzalez\equalcontrib
-}
-\affiliations{
- %Afiliations
- \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
- % If you have multiple authors and multiple affiliations
- % use superscripts in text and roman font to identify them.
- % For example,
-
- % Sunil Issar\textsuperscript{\rm 2},
- % J. Scott Penberthy\textsuperscript{\rm 3},
- % George Ferguson\textsuperscript{\rm 4},
- % Hans Guesgen\textsuperscript{\rm 5}
- % Note that the comma should be placed after the superscript
-
- 1900 Embarcadero Road, Suite 101\\
- Palo Alto, California 94303-3310 USA\\
- % email address must be in roman text type, not monospace or sans serif
- proceedings-questions@aaai.org
-%
-% See more examples next
-}
-
-%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
-\iffalse
-\title{My Publication Title --- Single Author}
-\author {
- Author Name
-}
-\affiliations{
- Affiliation\\
- Affiliation Line 2\\
- name@example.com
-}
-\fi
-
-\iffalse
-%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
-\title{My Publication Title --- Multiple Authors}
-\author {
- % Authors
- First Author Name\textsuperscript{\rm 1},
- Second Author Name\textsuperscript{\rm 2},
- Third Author Name\textsuperscript{\rm 1}
-}
-\affiliations {
- % Affiliations
- \textsuperscript{\rm 1}Affiliation 1\\
- \textsuperscript{\rm 2}Affiliation 2\\
- firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
-}
-\fi
-
-
-% REMOVE THIS: bibentry
-% This is only needed to show inline citations in the guidelines document. You should not need it and can safely delete it.
-\usepackage{bibentry}
-% END REMOVE bibentry
-
-\begin{document}
-
-\maketitle
-
- \begin{abstract}
- Counterfactual Explanations offer an intuitive and straightforward way to explain black-box models and offer Algorithmic Recourse to individuals. To address the need for plausible explanations, existing work has primarily relied on surrogate models to learn how the input data is distributed. This effectively reallocates the task of learning realistic explanations for the data from the model itself to the surrogate. Consequently, the generated explanations may seem plausible to humans but need not necessarily describe the behaviour of the black-box model faithfully. We formalise this notion of faithfulness through the introduction of a tailored evaluation metric and propose a novel algorithmic framework for generating \textbf{E}nergy-\textbf{C}onstrained \textbf{C}onformal \textbf{Co}unterfactuals (ECCCos) that are only as plausible as the model permits. Through extensive empirical studies, we demonstrate that ECCCos reconcile the need for faithfulness and plausibility. In particular, we show that for models with gradient access, it is possible to achieve state-of-the-art performance without the need for surrogate models. To do so, our framework relies solely on properties defining the black-box model itself by leveraging recent advances in Energy-Based Modelling and Conformal Prediction. To our knowledge, this is the first venture in this direction for generating faithful Counterfactual Explanations. Thus, we anticipate that ECCCos can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
- \end{abstract}
-
- \section{Introduction}\label{intro}
-
- Counterfactual Explanations (CE) provide a powerful, flexible and intuitive way to not only explain black-box models but also help affected individuals through the means of Algorithmic Recourse. Instead of opening the Black Box, CE works under the premise of strategically perturbing model inputs to understand model behaviour~\citep{wachter2017counterfactual}. Intuitively speaking, we generate explanations in this context by asking what-if questions of the following nature: `Our credit risk model currently predicts that this individual is not credit-worthy. What if they reduced their monthly expenditures by 10\%?'
-
- This is typically implemented by defining a target outcome $\mathbf{y}^+ \in \mathcal{Y}$ for some individual $\mathbf{x} \in \mathcal{X}=\mathbb{R}^D$ described by $D$ attributes, for which the model $M_{\theta}:\mathcal{X}\mapsto\mathcal{Y}$ initially predicts a different outcome: $M_{\theta}(\mathbf{x})\ne \mathbf{y}^+$. Counterfactuals are then searched by minimizing a loss function that compares the predicted model output to the target outcome: $\text{yloss}(M_{\theta}(\mathbf{x}),\mathbf{y}^+)$. Since CE work directly with the black-box model, valid counterfactuals always have full local fidelity by construction where fidelity is defined as the degree to which explanations approximate the predictions of a black-box model~\citep{mothilal2020explaining,molnar2020interpretable}.
-
- In situations where full fidelity is a requirement, CE offer a more appropriate solution to Explainable Artificial Intelligence (XAI) than other popular approaches like LIME~\citep{ribeiro2016why} and SHAP~\citep{lundberg2017unified}, which involve local surrogate models. But even full fidelity is not a sufficient condition for ensuring that an explanation faithfully describes the behaviour of a model. That is because multiple very distinct explanations can all lead to the same model prediction, especially when dealing with heavily parameterized models like deep neural networks, which are typically underspecified by the data~\citep{wilson2020case}.
-
- In the context of CE, the idea that no two explanations are the same arises almost naturally. A key focus in the literature has therefore been to identify those explanations and algorithmic recourses that are most appropriate based on a myriad of desiderata such as sparsity, actionability and plausibility. In this work, we draw closer attention to model faithfulness rather than fidelity as a desideratum for counterfactuals. Our key contributions are as follows:
-
- \begin{itemize}
- \item We show that fidelity is an insufficient evaluation metric for counterfactuals (Section~\ref{fidelity}) and propose a definition of faithfulness that gives rise to more suitable metrics (Section~\ref{faithfulness}).
- \item We introduce a novel algorithmic approach for generating Energy-Constrained Conformal Counterfactuals (ECCCos) in Section~\ref{meth}.
- \item We provide extensive empirical evidence demonstrating that ECCCos faithfully explain model behaviour and attain plausibility only when appropriate (Section~\ref{emp}).
- \end{itemize}
-
- To our knowledge, this is the first venture in this direction for generating faithful counterfactuals. Thus, we anticipate that ECCCos can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
-
- \section{Background}\label{background}
-
- While CE can also be generated for arbitrary regression models~\citep{spooner2021counterfactual}, existing work has primarily focused on classification problems. Let $\mathcal{Y}=(0,1)^K$ denote the one-hot-encoded output domain with $K$ classes. Then most counterfactual generators rely on gradient descent to optimize different flavours of the following counterfactual search objective:
-
- \begin{equation} \label{eq:general}
- \begin{aligned}
- \mathbf{Z}^\prime &= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \left\{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda {\text{cost}(f(\mathbf{Z}^\prime)) } \right\}
- \end{aligned}
- \end{equation}
-
- Here $\text{yloss}(\cdot)$ denotes the primary loss function, $f(\cdot)$ is a function that maps from the counterfactual state space to the feature space and $\text{cost}(\cdot)$ is either a single penalty or a collection of penalties that are used to impose constraints through regularization. Equation~\ref{eq:general} restates the baseline approach to gradient-based counterfactual search proposed by~\citet{wachter2017counterfactual} in general form as introduced by~\citet{altmeyer2023endogenous}. To explicitly account for the multiplicity of explanations, $\mathbf{Z}^\prime=\{ \mathbf{z}_l\}_L$ denotes an $L$-dimensional array of counterfactual states.
-
- The baseline approach, which we will simply refer to as \textit{Wachter}, searches a single counterfactual directly in the feature space and penalises its distance to the original factual. In this case, $f(\cdot)$ is simply the identity function and $\mathcal{Z}$ corresponds to the feature space itself. Many derivative works of~\citet{wachter2017counterfactual} have proposed new flavours of Equation~\ref{eq:general}, each of them designed to address specific \textit{desiderata} that counterfactuals ought to meet in order to properly serve both AI practitioners and individuals affected by algorithmic decision-making systems. The list of desiderata includes but is not limited to the following: sparsity, proximity~\citep{wachter2017counterfactual}, actionability~\citep{ustun2019actionable}, diversity~\citep{mothilal2020explaining}, plausibility~\citep{joshi2019realistic,poyiadzi2020face,schut2021generating}, robustness~\citep{upadhyay2021robust,pawelczyk2022probabilistically,altmeyer2023endogenous} and causality~\citep{karimi2021algorithmic}. Different counterfactual generators addressing these needs have been extensively surveyed and evaluated in various studies~\citep{verma2020counterfactual,karimi2020survey,pawelczyk2021carla,artelt2021evaluating,guidotti2022counterfactual}.
-
- Perhaps unsurprisingly, the different desiderata are often positively correlated. For example, \citet{artelt2021evaluating} find that plausibility typically also leads to improved robustness. Similarly, plausibility has also been connected to causality in the sense that plausible counterfactuals respect causal relationships~\citep{mahajan2020preserving}. Consequently, the plausibility of counterfactuals has been among the primary concerns for researchers. Achieving plausibility is equivalent to ensuring that the generated counterfactuals comply with the true and unobserved data-generating process (DGP). We define plausibility formally in this work as follows:
-
- \begin{definition}[Plausible Counterfactuals]
- \label{def:plausible}
- Let $\mathcal{X}|\mathbf{y}^+= p(\mathbf{x}|\mathbf{y}^+)$ denote the true conditional distribution of samples in the target class $\mathbf{y}^+$. Then for $\mathbf{x}^{\prime}$ to be considered a plausible counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}|\mathbf{y}^+$.
- \end{definition}
-
- To generate plausible counterfactuals, we need to be able to quantify the DGP: $\mathcal{X}|\mathbf{y}^+$. One straightforward way to do this is to use surrogate models for the task. \citet{joshi2019realistic}, for example, suggest that instead of searching counterfactuals in the feature space $\mathcal{X}$, we can instead traverse a latent embedding $\mathcal{Z}$ (Equation~\ref{eq:general}) that implicitly codifies the DGP. To learn the latent embedding, they propose using a generative model such as a Variational Autoencoder (VAE). Provided the surrogate model is well-specified, their proposed approach called \textit{REVISE} can yield plausible explanations. Others have proposed similar approaches: \citet{dombrowski2021diffeomorphic} traverse the base space of a normalizing flow to solve Equation~\ref{eq:general}; \citet{poyiadzi2020face} use density estimators ($\hat{p}: \mathcal{X} \mapsto [0,1]$) to constrain the counterfactuals to dense regions in the feature space; and, finally, \citet{karimi2021algorithmic} assume knowledge about the structural causal model that generates the data.
-
- A competing approach towards plausibility that is also closely related to this work instead relies on the black-box model itself. \citet{schut2021generating} show that to meet the plausibility objective we need not explicitly model the input distribution. Pointing to the undesirable engineering overhead induced by surrogate models, they propose that we rely on the implicit minimisation of predictive uncertainty instead. Their proposed methodology, which we will refer to as \textit{Schut}, solves Equation~\ref{eq:general} by greedily applying Jacobian-Based Saliency Map Attacks (JSMA) in the feature space with cross-entropy loss and no penalty at all. The authors demonstrate theoretically and empirically that their approach yields counterfactuals for which the model $M_{\theta}$ predicts the target label $\mathbf{y}^+$ with high confidence. Provided the model is well-specified, these counterfactuals are plausible. This idea hinges on the assumption that the black-box model provides well-calibrated predictive uncertainty estimates.
-
- \section{Why Fidelity is not Enough}\label{fidelity}
-
- As discussed in the introduction, any valid counterfactual also has full fidelity by construction: solutions to Equation~\ref{eq:general} are considered valid as soon as the label predicted by the model matches the target class. So while fidelity always applies, counterfactuals that address the various desiderata introduced above can look vastly different from each other.
-
- To demonstrate this with an example, we have trained a simple image classifier $M_{\theta}$ on the well-known \textit{MNIST} dataset~\citep{lecun1998mnist}: a Multi-Layer Perceptron (\textit{MLP}) with above 90 percent test accuracy. No measures have been taken to improve the model's adversarial robustness or its capacity for predictive uncertainty quantification. The far left panel of Figure ~\ref{fig:motiv} shows a random sample drawn from the dataset. The underlying classifier correctly predicts the label `nine' for this image. For the given factual image and model, we have used \textit{Wachter}, \textit{Schut} and \textit{REVISE} to generate one counterfactual each in the target class `seven'. The perturbed images are shown next to the factual image from left to right in Figure ~\ref{fig:motiv}. Captions on top of the individual images indicate the generator along with the predicted probability that the image belongs to the target class. In all three cases that probability is above 90 percent and yet the counterfactuals look very different from each other.
-
- \begin{figure}
- \centering
- \includegraphics[width=0.4\textwidth]{../artifacts/results/images/mnist_motivation.png}
- \caption{Counterfactuals for turning a 9 (nine) into a 7 (seven): original image (left); then from left to right the counterfactuals generated using \textit{Wachter}, \textit{Schut} and \textit{REVISE}.}\label{fig:motiv}
- \end{figure}
-
- Since \textit{Wachter} is only concerned with proximity, the generated counterfactual is almost indistinguishable from the factual. The approach by~\citet{schut2021generating} expects a well-calibrated model that can generate predictive uncertainty estimates. Since this is not the case, the generated counterfactual looks like an adversarial example. Finally, the counterfactual generated by \textit{REVISE} looks much more plausible than the other two. But is it also more faithful to the behaviour of our \textit{MNIST} classifier? That is much less clear because the surrogate used by \textit{REVISE} introduces friction: the generated explanations no longer depend exclusively on the black-box model itself.
-
- So which of the counterfactuals most faithfully explains the behaviour of our image classifier? Fidelity cannot help us to make that judgement, because all of these counterfactuals have full fidelity. Thus, fidelity is an insufficient evaluation metric to assess the faithfulness of CE.
-
- \section{A New Notion of Faithfulness}\label{faithfulness}
-
- Considering the limitations of fidelity as demonstrated in the previous section, analogous to Definition~\ref{def:plausible}, we introduce a new notion of faithfulness in the context of CE:
-
- \begin{definition}[Faithful Counterfactuals]
- \label{def:faithful}
- Let $\mathcal{X}_{\theta}|\mathbf{y}^+ = p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ denote the conditional distribution of $\mathbf{x}$ in the target class $\mathbf{y}^+$, where $\theta$ denotes the parameters of model $M_{\theta}$. Then for $\mathbf{x}^{\prime}$ to be considered a faithful counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}_{\theta}|\mathbf{y}^+$.
- \end{definition}
-
- In doing this, we merge in and nuance the concept of plausibility (Definition~\ref{def:plausible}) where the notion of `consistent with the data' becomes `consistent with what the model has learned about the data'.
-
- \subsection{Quantifying the Model's Generative Property}
-
- To assess counterfactuals with respect to Definition~\ref{def:faithful}, we need a way to quantify the posterior conditional distribution $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$. To this end, we draw on recent advances in Energy-Based Modelling (EBM), a subdomain of machine learning that is concerned with generative or hybrid modelling~\citep{grathwohl2020your,du2020implicit}. In particular, note that if we fix $\mathbf{y}$ to our target value $\mathbf{y}^+$, we can conditionally draw from $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ by randomly initializing $\mathbf{x}_0$ and then using Stochastic Gradient Langevin Dynamics (SGLD) as follows,
-
- \begin{equation}\label{eq:sgld}
- \begin{aligned}
- \mathbf{x}_{j+1} &\leftarrow \mathbf{x}_j - \frac{\epsilon^2}{2} \mathcal{E}(\mathbf{x}_j|\mathbf{y}^+) + \epsilon \mathbf{r}_j, && j=1,...,J
- \end{aligned}
- \end{equation}
-
- where $\mathbf{r}_j \sim \mathcal{N}(\mathbf{0},\mathbf{I})$ is the stochastic term and the step-size $\epsilon$ is typically polynomially decayed~\citep{welling2011bayesian}. The term $\mathcal{E}(\mathbf{x}_j|\mathbf{y}^+)$ denotes the model energy conditioned on the target class label $\mathbf{y}^+$ which we specify as the negative logit corresponding to the target class label $\mathbf{y}^*$. To allow for faster sampling, we follow the common practice of choosing the step-size $\epsilon$ and the standard deviation of $\mathbf{r}_j$ separately. While $\mathbf{x}_J$ is only guaranteed to distribute as $p_{\theta}(\mathbf{x}|\mathbf{y}^*)$ if $\epsilon \rightarrow 0$ and $J \rightarrow \infty$, the bias introduced for a small finite $\epsilon$ is negligible in practice \citep{murphy2023probabilistic,grathwohl2020your}. Appendix~\ref{app:jem} provides additional implementation details for any tasks related to energy-based modelling.
-
- Generating multiple samples using SGLD thus yields an empirical distribution $\hat{\mathbf{X}}_{\theta,\mathbf{y}^+}$ that approximates what the model has learned about the input data. While in the context of EBM, this is usually done during training, we propose to repurpose this approach during inference in order to evaluate and generate faithful model explanations.
-
- \subsection{Evaluating Plausibility and Faithfulness}
-
- The parallels between our definitions of plausibility and faithfulness imply that we can also use similar evaluation metrics in both cases. Since existing work has focused heavily on plausibility, it offers a useful starting point. In particular,~\citet{guidotti2022counterfactual} have proposed an implausibility metric that measures the distance of the counterfactual from its nearest neighbour in the target class. As this distance is reduced, counterfactuals get more plausible under the assumption that the nearest neighbour itself is plausible in the sense of Definition~\ref{def:plausible}. In this work, we use the following adapted implausibility metric,
-
- \begin{equation}\label{eq:impl}
- \begin{aligned}
- \text{impl}(\mathbf{x}^{\prime},\mathbf{X}_{\mathbf{y}^+}) = \frac{1}{\lvert\mathbf{X}_{\mathbf{y}^+}\rvert} \sum_{\mathbf{x} \in \mathbf{X}_{\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
- \end{aligned}
- \end{equation}
-
- where $\mathbf{x}^{\prime}$ denotes the counterfactual and $\mathbf{X}_{\mathbf{y}^+}$ is a subsample of the training data in the target class $\mathbf{y}^+$. By averaging over multiple samples in this manner, we avoid the risk that the nearest neighbour of $\mathbf{x}^{\prime}$ itself is not plausible according to Definition~\ref{def:plausible} (e.g an outlier).
-
- Equation~\ref{eq:impl} gives rise to a similar evaluation metric for unfaithfulness. We merely swap out the subsample of individuals in the target class for a subset $\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}$ of the generated conditional samples:
-
- \begin{equation}\label{eq:faith}
- \begin{aligned}
- \text{unfaith}(\mathbf{x}^{\prime},\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}) = \frac{1}{\lvert \hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}\rvert} \sum_{\mathbf{x} \in \hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
- \end{aligned}
- \end{equation}
-
- Specifically, we form this subset based on the $n_E$ generated samples with the lowest energy.
-
- \section{Energy-Constrained Conformal Counterfactuals}\label{meth}
-
- In this section, we describe \textit{ECCCo}, our proposed framework for generating Energy-Constrained Conformal Counterfactuals (ECCCos). It is based on the premise that counterfactuals should first and foremost be faithful. Plausibility, as a secondary concern, is then still attainable, but only to the degree that the black-box model itself has learned plausible explanations for the underlying data.
-
- We begin by stating our proposed objective function, which involves tailored loss and penalty functions that we will explain in the following. In particular, we extend Equation~\ref{eq:general} as follows:
-
- \begin{equation} \label{eq:eccco}
- \begin{aligned}
- \mathbf{Z}^\prime= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^M} &\{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda_{1} {\text{dist}(f(\mathbf{Z}^\prime),\mathbf{x}) } \\
- &+ \lambda_2 \text{unfaith}(f(\mathbf{Z}^\prime),\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}) + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
- \end{aligned}
- \end{equation}
-
- The first penalty term involving $\lambda_1$ induces proximity like in~\citet{wachter2017counterfactual}. Our default choice for $\text{dist}(\cdot)$ is the L1 Norm due to its sparsity-inducing properties. The second penalty term involving $\lambda_2$ induces faithfulness by constraining the energy of the generated counterfactual where $\text{unfaith}(\cdot)$ corresponds to the metric defined in Equation~\ref{eq:faith}. The third and final penalty term involving $\lambda_3$ introduces a new concept: it ensures that the generated counterfactual is associated with low predictive uncertainty. As mentioned above,~\citet{schut2021generating} have shown that plausible counterfactuals can be generated implicitly through predictive uncertainty minimization. Unfortunately, this relies on the assumption that the model itself can provide predictive uncertainty estimates, which may be too restrictive in practice.
-
- To relax this assumption, we leverage recent advances in Conformal Prediction (CP), an approach to predictive uncertainty quantification that has recently gained popularity~\citep{angelopoulos2021gentle,manokhin2022awesome}. Crucially for our intended application, CP is model-agnostic and can be applied during inference without placing any restrictions on model training. Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the training data or a dedicated calibration dataset. Conformal classifiers produce prediction sets for individual inputs that include all output labels that can be reasonably attributed to the input. These sets tend to be larger for inputs that do not conform with the training data and are characterized by high predictive uncertainty.
-
- In order to generate counterfactuals that are associated with low predictive uncertainty, we use a smooth set size penalty introduced by~\citet{stutz2022learning} in the context of conformal training:
-
- \begin{equation}\label{eq:setsize}
- \begin{aligned}
- \Omega(C_{\theta}(\mathbf{x};\alpha))&=\max \left(0, \sum_{\mathbf{y}\in\mathcal{Y}}C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha) - \kappa \right)
- \end{aligned}
- \end{equation}
-
- Here, $\kappa \in \{0,1\}$ is a hyper-parameter and $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha)$ can be interpreted as the probability of label $\mathbf{y}$ being included in the prediction set. In order to compute this penalty for any black-box model we merely need to perform a single calibration pass through a holdout set $\mathcal{D}_{\text{cal}}$. Arguably, data is typically abundant and in most applications, practitioners tend to hold out a test data set anyway. Consequently, CP removes the restriction on the family of predictive models, at the small cost of reserving a subset of the available data for calibration. This particular case of conformal prediction is referred to as Split Conformal Prediction (SCP) as it involves splitting the training data into a proper training dataset and a calibration dataset. In addition to the smooth set size penalty, we have also experimented with the use of a tailored function for $\text{yloss}(\cdot)$ that enforces that only the target label $\mathbf{y}^+$ is included in the prediction set ~\citet{stutz2022learning}. Further details are provided in Appendix~\ref{app:cp}.
-
- \begin{figure}
- \centering
- \includegraphics[width=0.5\textwidth]{../artifacts/results/images/poc_gradient_fields.png}
- \caption{Gradient fields and counterfactual paths for different generators. The objective is to generate a counterfual in the `blue' class for a sample from the `orange' class. Bright yellow stars indicate conditional samples generated through SGLD. The underlying classifier is a Joint Energy Model.}\label{fig:poc}
- \end{figure}
-
- \renewcommand{\algorithmicrequire}{\textbf{Input:}}
- \renewcommand{\algorithmicensure}{\textbf{Output:}}
-
- \begin{algorithm}
- \caption{The \textit{ECCCo} generator}\label{alg:eccco}
- \begin{algorithmic}[1]
- \Require $\mathbf{x}, \mathbf{y}^+, M_{\theta}, f, \Lambda=[\lambda_1,\lambda_2,\lambda_3], \alpha, \mathcal{D}, T, \eta, n_{\mathcal{B}}, n_E$ where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^+$
- \Ensure $\mathbf{x}^\prime$
- \State Initialize $\mathbf{z}^\prime \gets f^{-1}(\mathbf{x})$ \Comment{Map to counterfactual state space.}
- \State Generate $\left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}} \gets p_{\theta}(\mathbf{x}_{\mathbf{y}^+})$ \Comment{Generate $n_{\mathcal{B}}$ samples using SGLD (Equation~\ref{eq:sgld}).}
- \State Store $\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+} \gets \left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}}$ \Comment{Choose $n_E$ lowest-energy samples.}
- \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$ \Comment{Calibrate model through Split Conformal Prediction.}
- \State Initialize $t \gets 0$
- \While{\textit{not converged} or $t < T$} \Comment{For convergence conditions see Appendix~\ref{app:eccco}.}
- \State $\mathbf{z}^\prime \gets \mathbf{z}^\prime - \eta \nabla_{\mathbf{z}^\prime} \mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ \Comment{Take gradient step of size $\eta$.}
- \State $t \gets t+1$
- \EndWhile
- \State $\mathbf{x}^\prime \gets f(\mathbf{z}^\prime)$ \Comment{Map back to feature space.}
- \end{algorithmic}
- \end{algorithm}
-
- To provide some further intuition about our objective defined in Equation~\ref{eq:eccco}, Figure~\ref{fig:poc} illustrates how the different components affect the counterfactual search for a synthetic dataset. The underlying classifier is a Joint Energy Model (\textit{JEM}) that was trained to predict the output class (`blue' or `orange') and generate class-conditional samples~\citep{grathwohl2020your}. We have used four different generator flavours to produce a counterfactual in the `blue' class for a sample from the `orange' class: \textit{Wachter}, which only uses the first penalty ($\lambda_2=\lambda_3=0$); \textit{ECCCo (no EBM)}, which does not constrain energy ($\lambda_2=0$); \textit{ECCCo (no CP)}, which involves no set size penalty ($\lambda_3=0$); and, finally, \textit{ECCCo}, which involves all penalties defined in Equation~\ref{eq:eccco}. Arrows indicate (negative) gradients with respect to the objective function at different points in the feature space.
-
- While \textit{Wachter} generates a valid counterfactual, it ends up close to the original starting point consistent with its objective. \textit{ECCCo (no EBM)} pushes the counterfactual further into the target domain to minimize predictive uncertainty, but the outcome is still not plausible. The counterfactual produced by \textit{ECCCo (no CP)} is attracted by the generated samples shown in bright yellow. Since the \textit{JEM} has learned the conditional input distribution reasonably well in this case, the counterfactuals are both faithful and plausible. Finally, the outcome for \textit{ECCCo} looks similar, but the additional smooth set size penalty leads to somewhat faster convergence.
-
- Algorithm~\ref{alg:eccco} describes how exactly \textit{ECCCo} works. For the sake of simplicity and without loss of generality, we limit our attention to generating a single counterfactual $\mathbf{x}^\prime=f(\mathbf{z}^\prime)$. The counterfactual state $\mathbf{z}^\prime$ is initialized by passing the factual $\mathbf{x}$ through a simple feature transformer $f^{-1}$. Next, we generate $n_{\mathcal{B}}$ conditional samples $\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}$ using SGLD (Equation~\ref{eq:sgld}) and store the $n_E$ instances with the lowest energy. We then calibrate the model $M_{\theta}$ through Split Conformal Prediction. Finally, we search counterfactuals through gradient descent where $\mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ denotes our loss function defined in Equation~\ref{eq:eccco}. The search terminates once the convergence criterium is met or the maximum number of iterations $T$ has been exhausted. Note that the choice of convergence criterium has important implications on the final counterfactual which we explain in Appendix~\ref{app:eccco}.
-
- \section{Empirical Analysis}\label{emp}
-
- Our goal in this section is to shed light on the following research questions:
-
- \begin{question}[Faithfulness]\label{rq:faithfulness}
- Are ECCCos more faithful than counterfactuals produced by our benchmark generators?
- \end{question}
-
- \begin{question}[Balancing Objectives]\label{rq:plausibility}
- Compared to our benchmark generators, how do ECCCos balance the two key objectives of faithfulness and plausibility?
- \end{question}
-
- The second question is motivated by the intuition that faithfulness and plausibility should coincide for models that have learned plausible explanations of the data. Next, we first briefly describe our experimental setup before presenting our main results.
-
- \subsection{Experimental Setup}
-
- To assess and benchmark the performance of our proposed generator against the state of the art, we generate multiple counterfactuals for different models and datasets. In particular, we compare \textit{ECCCo} and its variants to the following counterfactual generators that were introduced above: firstly; \textit{Schut}, which works under the premise of minimizing predictive uncertainty; secondly, \textit{REVISE}, which is state-of-the-art with respect to plausibility; and, finally, \textit{Wachter}, which serves as our baseline.
-
- We use both synthetic and real-world datasets from different domains, all of which are publicly available and commonly used to train and benchmark classification algorithms. We synthetically generate a dataset containing two \textit{Linearly Separable} Gaussian clusters ($n=1000$), as well as the well-known \textit{Circles} ($n=1000$) and \textit{Moons} ($n=2500$) data. Since these data are generated by distributions of varying degrees of complexity, they allow us to assess how the generators and our proposed evaluation metrics handle this.
-
- As for real-world data, we follow~\citet{schut2021generating} and use the \textit{MNIST}~\citep{lecun1998mnist} dataset containing images of handwritten digits such as the example shown above in Figure~\ref{fig:motiv}. From the social sciences domain, we include Give Me Some Credit (\textit{GMSC})~\citep{kaggle2011give}: a tabular dataset that has been studied extensively in the literature on Algorithmic Recourse~\citep{pawelczyk2021carla}. It consists of 11 numeric features that can be used to predict the binary outcome variable indicating whether retail borrowers experience financial distress.
-
- For the predictive modelling tasks, we use simple neural networks (\textit{MLP}) and Joint Energy Models (\textit{JEM}). For the more complex real-world datasets we also use ensembling in each case. Both joint-energy modelling and ensembling have been associated with improved generative properties and adversarial robustness~\citep{grathwohl2020your,lakshminarayanan2016simple}, so we expect this to be positively correlated with the plausibility of ECCCos. To account for stochasticity, we generate multiple counterfactuals for each target class, generator, model and dataset. Specifically, we randomly sample $n^{-}$ times from the subset of individuals for which the given model predicts the non-target class $\mathbf{y}^{-}$ given the current target. We set $n^{-}=25$ for all of our synthetic datasets, $n^{-}=10$ for \textit{GMSC} and $n^{-}=5$ for \textit{MNIST}. Full details concerning our parameter choices, training procedures and model performance can be found in Appendix~\ref{app:setup}.
-
- \subsection{Results for Synthetic Data}
-
- Table~\ref{tab:results-synthetic} shows the key results for the synthetic datasets separated by model (first column) and generator (second column). The numerical columns show sample averages and standard deviations of our key evaluation metrics computed across all counterfactuals. We have highlighted the best outcome for each model and metric in bold. To provide some sense of effect sizes, we have added asterisks to indicate that a given value is at least one ($*$) or two ($**$) standard deviations lower than the baseline (\textit{Wachter}).
-
- Starting with the high-level results for our \textit{Linearly Separable} data, we find that \textit{ECCCo} produces the most faithful counterfactuals for both black-box models. This is consistent with our design since \textit{ECCCo} directly enforces faithfulness through regularization. Crucially though, \textit{ECCCo} also produces the most plausible counterfactuals for both models. This dataset is so simple that even the \textit{MLP} has learned plausible explanations of the input data. Zooming in on the granular details for the \textit{Linearly Separable} data, the results for \textit{ECCCo (no CP)} and \textit{ECCCo (no EBM)} indicate that the positive results are dominated by the effect of quantifying and leveraging the model's generative property (EBM). Conformal Prediction alone only leads to marginally improved faithfulness and plausibility.
-
- The findings for the \textit{Moons} dataset are broadly in line with the findings so far: for the \textit{JEM}, \textit{ECCCo} yields substantially more faithful and plausible counterfactuals than all other generators. For the \textit{MLP}, faithfulness is maintained but counterfactuals are not plausible. This high-level pattern is broadly consistent with other more complex datasets and supportive of our narrative, so it is worth highlighting: ECCCos consistently achieve high faithfulness, which---subject to the quality of the model itself---coincides with high plausibility. By comparison, \textit{REVISE} yields the most plausible counterfactuals for the \textit{MLP}, but it does so at the cost of faithfulness. We also observe that the best results for \textit{ECCCo} are achieved when using both penalties. Once again though, the generative component (EBM) has a stronger impact on the positive results for the \textit{JEM}.
-
- For the \textit{Circles} data, it appears that \textit{REVISE} performs well, but we note that it generates valid counterfactuals only half of the time (see Appendix~\ref{app:results} for a complete overview including additional common evaluation metrics). The underlying VAE with default parameters has not adequately learned the data-generating process. Of course, it is possible to improve generative performance through hyperparameter tuning but this example serves to illustrate that \textit{REVISE} depends on the quality of its surrogate. Independent of the outcome for \textit{REVISE}, however, the results do not seem to indicate that \textit{ECCCo} substantially improves faithfulness and plausibility for the \textit{Circles} data. We think this points to a limitation of our evaluation metrics rather than \textit{ECCCo} itself: computing average distances fails to account for the `wraparound' effect associated with circular data~\citep{gill2010circular}.
-
- \import{contents/}{table-synthetic.tex}
-
- \subsection{Results for Real-World Data}
-
- The results for our real-world datasets are shown in Table~\ref{tab:results-real-world}. Once again the findings indicate that the plausibility of ECCCos is positively correlated with the capacity of the black-box model to distinguish plausible from implausible inputs. The case is very clear for \textit{MNIST}: ECCCos are consistently more faithful than the counterfactuals produced by our benchmark generators and their plausibility gradually improves through ensembling and joint-energy modelling. Interestingly, faithfulness also gradually improves for \textit{REVISE}. This indicates that as our models improve, their generative capacity approaches that of the surrogate VAE used by \textit{REVISE}. The VAE still outperforms our classifiers in this regard, as evident from the fact that \textit{ECCCo} never quite reaches the same level of plausibility as \textit{REVISE}. With reference to Appendix~\ref{app:results} we note that the results for \textit{Schut} need to be discounted as it rarely produces valid counterfactuals for \textit{MNIST}. Relatedly, we find that \textit{ECCCo} is the only generator that consistently achieves full validity. Finally, it is worth noting that \textit{ECCCo} produces counterfactual images with the lowest average predictive uncertainty for all models.
-
- For the tabular credit dataset (\textit{GMSC}) it is inherently challenging to use deep neural networks in order to achieve good discriminative performance~\citep{borisov2021deep,grinsztajn2022why} and generative performance~\citep{liu2023goggle}, respectively. In order to achieve high plausibility, \textit{ECCCo} effectively requires classifiers to achieve good performance for both tasks. Since this is a challenging task even for Joint Energy Models, it is not surprising to find that even though \textit{ECCCo} once again achieves state-of-the-art faithfulness, it is outperformed by \textit{REVISE} and \textit{Schut} with respect to plausibility.
-
- \subsection{Key Takeways}
-
- To conclude this section, we summarize our findings with reference to the opening questions. The results clearly demonstrate that \textit{ECCCo} consistently achieves state-of-the-art faithfulness, as it was designed to do (Research Question~\ref{rq:faithfulness}). A related important finding is that \textit{ECCCo} yields highly plausible explanations provided that they faithfully describe model behaviour (Research Question~\ref{rq:plausibility}). \textit{ECCCo} achieves this result primarily by leveraging the model's generative property.
-
- \import{contents/}{table-real-world.tex}
-
- \section{Limitations}
-
- Even though we have taken considerable measures to study our proposed methodology carefully, limitations can still be identified. In particular, we have found that the performance of \textit{ECCCo} is sensitive to hyperparameter choices. In order to achieve faithfulness, we generally had to penalise the distance from generated samples slightly more than the distance from factual values.
-
- Conversely, we have not found that strongly penalising prediction set sizes had any discernable effect. Our results indicate that CP alone is often not sufficient to achieve faithfulness and plausibility, although we acknowledge that this needs to be investigated more thoroughly through future work.
-
- While our approach is readily applicable to models with gradient access like deep neural networks, more work is needed to generalise it to other machine learning models such as decision trees. Relatedly, common challenges associated with Energy-Based Modelling including sensitivity to scale, training instabilities and sensitivity to hyperparameters also apply to \textit{ECCCo}.
-
- \section{Conclusion}
-
- This work leverages recent advances in Energy-Based Modelling and Conformal Prediction in the context of Explainable Artificial Intelligence. We have proposed a new way to generate counterfactuals that are maximally faithful to the black-box model they aim to explain. Our proposed generator, \textit{ECCCo}, produces plausible counterfactuals if and only if the black-box model itself has learned realistic explanations for the data, which we have demonstrated through rigorous empirical analysis. This should enable researchers and practitioners to use counterfactuals in order to discern trustworthy models from unreliable ones. While the scope of this work limits its generalizability, we believe that \textit{ECCCo} offers a solid baseline for future work on faithful Counterfactual Explanations.
-
- \section{Acknowledgments}
-
- Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
- collaboration.
-
- \bibliography{bib}
-
-\end{document}
diff --git a/AAAI/template/aaai24.bib b/AAAI/template/aaai24.bib
deleted file mode 100644
index 7b7d2bcf44a7488d282c3e9b1f9079598dc99fa3..0000000000000000000000000000000000000000
--- a/AAAI/template/aaai24.bib
+++ /dev/null
@@ -1,111 +0,0 @@
-@book{em:86,
- editor = "Engelmore, Robert and Morgan, Anthony",
- title = "Blackboard Systems",
- year = 1986,
- address = "Reading, Mass.",
- publisher = "Addison-Wesley",
-}
-
-@inproceedings{c:83,
- author = "Clancey, William J.",
- year = 1983,
- title = "{Communication, Simulation, and Intelligent
-Agents: Implications of Personal Intelligent Machines
-for Medical Education}",
- booktitle="Proceedings of the Eighth International Joint Conference on Artificial Intelligence {(IJCAI-83)}",
- pages = "556-560",
- address = "Menlo Park, Calif",
- publisher = "{IJCAI Organization}",
-}
-@inproceedings{c:84,
- author = "Clancey, William J.",
- year = 1984,
- title = "{Classification Problem Solving}",
- booktitle = "Proceedings of the Fourth National
- Conference on Artificial Intelligence",
- pages = "45-54",
- address = "Menlo Park, Calif.",
- publisher="AAAI Press",
-}
-@article{r:80,
- author = {Robinson, Arthur L.},
- title = {New Ways to Make Microcircuits Smaller},
- volume = {208},
- number = {4447},
- pages = {1019--1022},
- year = {1980},
- doi = {10.1126/science.208.4447.1019},
- publisher = {American Association for the Advancement of Science},
- issn = {0036-8075},
- URL = {https://science.sciencemag.org/content/208/4447/1019},
- eprint = {https://science.sciencemag.org/content/208/4447/1019.full.pdf},
- journal = {Science},
-}
-@article{r:80x,
- author = "Robinson, Arthur L.",
- year = 1980,
- title = "{New Ways to Make Microcircuits Smaller---Duplicate Entry}",
- journal = "Science",
- volume = 208,
- pages = "1019-1026",
-}
-@article{hcr:83,
-title = {Strategic explanations for a diagnostic consultation system},
-journal = {International Journal of Man-Machine Studies},
-volume = {20},
-number = {1},
-pages = {3-19},
-year = {1984},
-issn = {0020-7373},
-doi = {https://doi.org/10.1016/S0020-7373(84)80003-6},
-url = {https://www.sciencedirect.com/science/article/pii/S0020737384800036},
-author = {Diane Warner Hasling and William J. Clancey and Glenn Rennels},
-abstract = {This article examines the problem of automatte explanation of reasoning, especially as it relates to expert systems. By explanation we mean the ability of a program to discuss what it is doing in some understandable way. We first present a general framework in which to view explanation and review some of the research done in this area. We then focus on the explanation system for NEOMYCIN, a medical consultation program. A consultation program interactively helps a user to solve a problem. Our goal is to have NEOMYCIN explain its problem-solving strategies. An explanation of strategy describes the plan the program is using to reach a solution. Such an explanation is usually concrete, referring to aspects of the current problem situation. Abstract explanations articulate a general principle, which can be applied in different situations; such explanations are useful in teaching and in explaining by analogy. We describe the aspects of NEOMYCIN that make abstract strategic explanations possible—the representation of strategic knowledge explicitly and separately from domain knowledge— and demonstrate how this representation can be used to generate explanations.}
-}
-@article{hcrt:83,
- author = "Hasling, Diane Warner and Clancey, William J. and Rennels, Glenn R. and Test, Thomas",
- year = 1983,
- title = "{Strategic Explanations in Consultation---Duplicate}",
- journal = "The International Journal of Man-Machine Studies",
- volume = 20,
- number = 1,
- pages = "3-19",
-}
-@techreport{r:86,
- author = "Rice, James",
- year = 1986,
- title = "{Poligon: A System for Parallel Problem Solving}",
- type = "Technical Report",
- number = "KSL-86-19",
- institution = "Dept.\ of Computer Science, Stanford Univ.",
-}
-@phdthesis{c:79,
- author = "Clancey, William J.",
- year = 1979,
- title = "{Transfer of Rule-Based Expertise
-through a Tutorial Dialogue}",
- type = "{Ph.D.} diss.",
- school = "Dept.\ of Computer Science, Stanford Univ.",
- address = "Stanford, Calif.",
-}
-@unpublished{c:21,
- author = "Clancey, William J.",
- title = "{The Engineering of Qualitative Models}",
- year = 2021,
- note = "Forthcoming",
-}
-@misc{c:22,
- title={Attention Is All You Need},
- author={Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
- year={2017},
- eprint={1706.03762},
- archivePrefix={arXiv},
- primaryClass={cs.CL}
-}
-@misc{c:23,
- title = "Pluto: The 'Other' Red Planet",
- author = "{NASA}",
- howpublished = "\url{https://www.nasa.gov/nh/pluto-the-other-red-planet}",
- year = 2015,
- note = "Accessed: 2018-12-06"
-}
\ No newline at end of file
diff --git a/AAAI/aaai24.bst b/paper/aaai/aaai24.bst
similarity index 100%
rename from AAAI/aaai24.bst
rename to paper/aaai/aaai24.bst
diff --git a/AAAI/aaai24.sty b/paper/aaai/aaai24.sty
similarity index 100%
rename from AAAI/aaai24.sty
rename to paper/aaai/aaai24.sty
diff --git a/AAAI/bib.bib b/paper/aaai/bib.bib
similarity index 100%
rename from AAAI/bib.bib
rename to paper/aaai/bib.bib
diff --git a/paper/aaai/paper.pdf b/paper/aaai/paper.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..3b7af7834d3f31d58af754923c9a4e21b5121723
Binary files /dev/null and b/paper/aaai/paper.pdf differ
diff --git a/paper/aaai/paper.tex b/paper/aaai/paper.tex
new file mode 100644
index 0000000000000000000000000000000000000000..f7872babe5134eeba42ca8fadf611ac3d06f48db
--- /dev/null
+++ b/paper/aaai/paper.tex
@@ -0,0 +1,201 @@
+%File: anonymous-submission-latex-2024.tex
+\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
+\usepackage[submission]{aaai24} % DO NOT CHANGE THIS
+\usepackage{times} % DO NOT CHANGE THIS
+\usepackage{helvet} % DO NOT CHANGE THIS
+\usepackage{courier} % DO NOT CHANGE THIS
+\usepackage[hyphens]{url} % DO NOT CHANGE THIS
+\usepackage{graphicx} % DO NOT CHANGE THIS
+\urlstyle{rm} % DO NOT CHANGE THIS
+\def\UrlFont{\rm} % DO NOT CHANGE THIS
+\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\frenchspacing % DO NOT CHANGE THIS
+\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
+\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
+%
+% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
+\usepackage{algorithm}
+% \usepackage{algorithmic}
+
+%
+% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
+% \usepackage{newfloat}
+% \usepackage{listings}
+% \DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
+% \lstset{%
+% basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
+% numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
+% aboveskip=0pt,belowskip=0pt,%
+% showstringspaces=false,tabsize=2,breaklines=true}
+% \floatstyle{ruled}
+% \newfloat{listing}{tb}{lst}{}
+% \floatname{listing}{Listing}
+%
+% Keep the \pdfinfo as shown here. There's no need
+% for you to add the /Title and /Author tags.
+\pdfinfo{
+/TemplateVersion (2024.1)
+}
+
+\usepackage{amsfonts} % blackboard math symbols
+\usepackage{amsmath}
+\usepackage{amsthm}
+\usepackage{caption}
+\usepackage{graphicx}
+\usepackage{algpseudocode}
+\usepackage{import}
+\usepackage{booktabs}
+\usepackage{longtable}
+\usepackage{array}
+\usepackage{multirow}
+
+
+% Numbered Environments:
+\newtheorem{definition}{Definition}[section]
+\newtheorem{question}{Research Question}[section]
+
+% Bibliography
+% \bibliographystyle{unsrtnat}
+% \setcitestyle{numbers,square,comma}
+
+% Algorithm
+\renewcommand{\algorithmicrequire}{\textbf{Input:}}
+\renewcommand{\algorithmicensure}{\textbf{Output:}}
+
+% DISALLOWED PACKAGES
+% \usepackage{authblk} -- This package is specifically forbidden
+% \usepackage{balance} -- This package is specifically forbidden
+% \usepackage{color (if used in text)
+% \usepackage{CJK} -- This package is specifically forbidden
+% \usepackage{float} -- This package is specifically forbidden
+% \usepackage{flushend} -- This package is specifically forbidden
+% \usepackage{fontenc} -- This package is specifically forbidden
+% \usepackage{fullpage} -- This package is specifically forbidden
+% \usepackage{geometry} -- This package is specifically forbidden
+% \usepackage{grffile} -- This package is specifically forbidden
+% \usepackage{hyperref} -- This package is specifically forbidden
+% \usepackage{navigator} -- This package is specifically forbidden
+% (or any other package that embeds links such as navigator or hyperref)
+% \indentfirst} -- This package is specifically forbidden
+% \layout} -- This package is specifically forbidden
+% \multicol} -- This package is specifically forbidden
+% \nameref} -- This package is specifically forbidden
+% \usepackage{savetrees} -- This package is specifically forbidden
+% \usepackage{setspace} -- This package is specifically forbidden
+% \usepackage{stfloats} -- This package is specifically forbidden
+% \usepackage{tabu} -- This package is specifically forbidden
+% \usepackage{titlesec} -- This package is specifically forbidden
+% \usepackage{tocbibind} -- This package is specifically forbidden
+% \usepackage{ulem} -- This package is specifically forbidden
+% \usepackage{wrapfig} -- This package is specifically forbidden
+% DISALLOWED COMMANDS
+% \nocopyright -- Your paper will not be published if you use this command
+% \addtolength -- This command may not be used
+% \balance -- This command may not be used
+% \baselinestretch -- Your paper will not be published if you use this command
+% \clearpage -- No page breaks of any kind may be used for the final version of your paper
+% \columnsep -- This command may not be used
+% \newpage -- No page breaks of any kind may be used for the final version of your paper
+% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
+% \pagestyle -- This command may not be used
+% \tiny -- This is not an acceptable font size.
+% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
+% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
+
+\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
+
+% The file aaai24.sty is the style file for AAAI Press
+% proceedings, working notes, and technical reports.
+%
+
+% Title
+
+% Your title must be in mixed case, not sentence case.
+% That means all verbs (including short verbs like be, is, using,and go),
+% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
+% articles, conjunctions, and prepositions are lower case unless they
+% directly follow a colon or long dash
+\title{AAAI Press Anonymous Submission\\Instructions for Authors Using \LaTeX{}}
+\author{
+ %Authors
+ % All authors must be in the same font size and format.
+ Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
+ AAAI Style Contributions by Pater Patel Schneider,
+ Sunil Issar,\\
+ J. Scott Penberthy,
+ George Ferguson,
+ Hans Guesgen,
+ Francisco Cruz\equalcontrib,
+ Marc Pujol-Gonzalez\equalcontrib
+}
+\affiliations{
+ %Afiliations
+ \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
+ % If you have multiple authors and multiple affiliations
+ % use superscripts in text and roman font to identify them.
+ % For example,
+
+ % Sunil Issar\textsuperscript{\rm 2},
+ % J. Scott Penberthy\textsuperscript{\rm 3},
+ % George Ferguson\textsuperscript{\rm 4},
+ % Hans Guesgen\textsuperscript{\rm 5}
+ % Note that the comma should be placed after the superscript
+
+ 1900 Embarcadero Road, Suite 101\\
+ Palo Alto, California 94303-3310 USA\\
+ % email address must be in roman text type, not monospace or sans serif
+ proceedings-questions@aaai.org
+%
+% See more examples next
+}
+
+%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+\iffalse
+\title{My Publication Title --- Single Author}
+\author {
+ Author Name
+}
+\affiliations{
+ Affiliation\\
+ Affiliation Line 2\\
+ name@example.com
+}
+\fi
+
+\iffalse
+%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+\title{My Publication Title --- Multiple Authors}
+\author {
+ % Authors
+ First Author Name\textsuperscript{\rm 1},
+ Second Author Name\textsuperscript{\rm 2},
+ Third Author Name\textsuperscript{\rm 1}
+}
+\affiliations {
+ % Affiliations
+ \textsuperscript{\rm 1}Affiliation 1\\
+ \textsuperscript{\rm 2}Affiliation 2\\
+ firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
+}
+\fi
+
+\begin{document}
+
+% Body of the paper
+\import{../}{body.tex}
+
+\section{Acknowledgments}
+
+Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
+collaboration.
+
+\bibliography{../bib}
+
+\pagebreak
+
+\onecolumn
+
+\import{../}{appendix.tex}
+
+\end{document}
diff --git a/AAAI/aaai24.bib b/paper/aaai/template/aaai24.bib
similarity index 100%
rename from AAAI/aaai24.bib
rename to paper/aaai/template/aaai24.bib
diff --git a/AAAI/template/aaai24.bst b/paper/aaai/template/aaai24.bst
similarity index 100%
rename from AAAI/template/aaai24.bst
rename to paper/aaai/template/aaai24.bst
diff --git a/AAAI/template/aaai24.sty b/paper/aaai/template/aaai24.sty
similarity index 100%
rename from AAAI/template/aaai24.sty
rename to paper/aaai/template/aaai24.sty
diff --git a/AAAI/template/anonymous-submission-latex-2024.pdf b/paper/aaai/template/anonymous-submission-latex-2024.pdf
similarity index 100%
rename from AAAI/template/anonymous-submission-latex-2024.pdf
rename to paper/aaai/template/anonymous-submission-latex-2024.pdf
diff --git a/AAAI/template/anonymous-submission-latex-2024.tex b/paper/aaai/template/anonymous-submission-latex-2024.tex
similarity index 100%
rename from AAAI/template/anonymous-submission-latex-2024.tex
rename to paper/aaai/template/anonymous-submission-latex-2024.tex
diff --git a/AAAI/template/figure1.pdf b/paper/aaai/template/figure1.pdf
similarity index 100%
rename from AAAI/template/figure1.pdf
rename to paper/aaai/template/figure1.pdf
diff --git a/AAAI/template/figure2.pdf b/paper/aaai/template/figure2.pdf
similarity index 100%
rename from AAAI/template/figure2.pdf
rename to paper/aaai/template/figure2.pdf
diff --git a/paper/appendix.tex b/paper/appendix.tex
new file mode 100644
index 0000000000000000000000000000000000000000..68abf60c5fc12cba9593bb83ec376b786c6b042d
--- /dev/null
+++ b/paper/appendix.tex
@@ -0,0 +1,156 @@
+\appendix
+\section*{Appendices}
+\renewcommand{\thesubsection}{\Alph{subsection}}
+
+The following appendices provide additional details that are relevant to the paper. Appendices~\ref{app:jem} and~\ref{app:cp} explain any tasks related to Energy-Based Modelling and Predictive Uncertainty Quantification through Conformal Prediction, respectively. Appendix~\ref{app:eccco} provides additional technical and implementation details about our proposed generator, \textit{ECCCo}, including references to our open-sourced code base. A complete overview of our experimental setup detailing our parameter choices, training procedures and initial black-box model performance can be found in Appendix~\ref{app:setup}. Finally, Appendix~\ref{app:results} reports all of our experimental results in more detail.
+
+\subsection{Energy-Based Modelling}\label{app:jem}
+
+Since we were not able to identify any existing open-source software for Energy-Based Modelling that would be flexible enough to cater to our needs, we have developed a \texttt{Julia} package from scratch. The package has been open-sourced, but to avoid compromising the double-blind review process, we refrain from providing more information at this stage. In our development we have heavily drawn on the existing literature:~\citet{du2020implicit} describe best practices for using EBM for generative modelling;~\citet{grathwohl2020your} explain how EBM can be used to train classifiers jointly for the discriminative and generative tasks. We have used the same package for training and inference, but there are some important differences between the two cases that are worth highlighting here.
+
+\subsubsection{Training: Joint Energy Models}
+
+To train our Joint Energy Models we broadly follow the approach outlined in~\citet{grathwohl2020your}. These models are trained to optimize a hybrid objective that involves a standard classification loss component $L_{\text{clf}}(\theta)=-\log p_{\theta}(\mathbf{y}|\mathbf{x})$ (e.g. cross-entropy loss) as well as a generative loss component $L_{\text{gen}}(\theta)=-\log p_{\theta}(\mathbf{x})$.
+
+To draw samples from $p_{\theta}(\mathbf{x})$, we rely exclusively on the conditional sampling approach described in~\citet{grathwohl2020your} for both training and inference: we first draw $\mathbf{y}\sim p(\mathbf{y})$ and then sample $\mathbf{x} \sim p_{\theta}(\mathbf{x}|\mathbf{y})$~\citep{grathwohl2020your} via Equation~\ref{eq:sgld} with energy $\mathcal{E}(\mathbf{x}|\mathbf{y})=\mu_{\theta}(\mathbf{x})[\mathbf{y}]$ where $\mu_{\theta}: \mathcal{X} \mapsto \mathbb{R}^K$ returns the linear predictions (logits) of our classifier $M_{\theta}$. While our package also supports unconditional sampling, we found conditional sampling to work well. It is also well aligned with CE, since in this context we are interested in conditioning on the target class.
+
+As mentioned in the body of the paper, we rely on a biased sampler involving separately specified values for the step size $\epsilon$ and the standard deviation $\sigma$ of the stochastic term involving $\mathbf{r}$. Formally, our biased sampler performs updates as follows:
+
+\begin{equation}\label{eq:biased-sgld}
+ \begin{aligned}
+ \hat{\mathbf{x}}_{j+1} &\leftarrow \hat{\mathbf{x}}_j - \frac{\epsilon}{2} \mathcal{E}(\hat{\mathbf{x}}_j|\mathbf{y}^+) + \sigma \mathbf{r}_j, && j=1,...,J
+ \end{aligned}
+\end{equation}
+
+Consistent with~\citet{grathwohl2020your}, we have specified $\epsilon=2$ and $\sigma=0.01$ as the default values for all of our experiments. The number of total SGLD steps $J$ varies by dataset (Table~\ref{tab:ebmparams}). Following best practices, we initialize $\mathbf{x}_0$ randomly in 5\% of all cases and sample from a buffer in all other cases. The buffer itself is randomly initialised and gradually grows to a maximum of 10,000 samples during training as $\hat{\mathbf{x}}_{J}$ is stored in each epoch~\citep{du2020implicit,grathwohl2020your}.
+
+It is important to realise that sampling is done during each training epoch, which makes training Joint Energy Models significantly harder than conventional neural classifiers. In each epoch the generated (batch of) sample(s) $\hat{\mathbf{x}}_{J}$ is used as part of the generative loss component, which compares its energy to that of observed samples $\mathbf{x}$: $L_{\text{gen}}(\theta)=\mu_{\theta}(\mathbf{x})[\mathbf{y}]-\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]$. Our full training objective can be summarized as follows,
+
+\begin{equation}\label{eq:jem-loss}
+ \begin{aligned}
+ L(\theta) &= L_{\text{clf}}(\theta) + L_{\text{gen}}(\theta) + \lambda L_{\text{reg}}(\theta)
+ \end{aligned}
+\end{equation}
+
+where $L_{\text{reg}}(\theta)$ is a Ridge penalty (L2 norm) that regularises energy magnitudes for both observed and generated samples~\citep{du2020implicit}. We have used varying degrees of regularization depending on the dataset ($\lambda$ in Table~\ref{tab:ebmparams}).
+
+Contrary to existing work, we have not typically used the entire minibatch of training data for the generative loss component but found that using a subset of the minibatch was often sufficient in attaining decent generative performance (Table~\ref{tab:ebmparams}). This has helped to reduce the computational burden for our models, which should make it easier for others to reproduce our findings. Figures~\ref{fig:mnist-gen} and~\ref{fig:moons-gen} show generated samples for our \textit{MNIST} and \textit{Moons} data, to provide a sense of their generative property.
+
+\import{contents/}{table_ebm_params.tex}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.75\linewidth]{../artifacts/results/images/mnist_generated_JEM Ensemble.png}
+ \caption{Conditionally generated \textit{MNIST} images for our JEM Ensemble.}\label{fig:mnist-gen}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.5\linewidth]{../artifacts/results/images/moons_generated_JEM.png}
+ \caption{Conditionally generated samples (stars) for our \textit{Moons} data using a JEM.}\label{fig:moons-gen}
+\end{figure}
+\subsubsection{Inference: Quantifying Models' Generative Property}
+
+At inference time, we assume no prior knowledge about the model's generative property. This means that we do not tab into the existing buffer of generated samples for our Joint Energy Models, but instead generate conditional samples from scratch. While we have relied on the default values $\epsilon=2$ and $\sigma=0.01$ also during inference, the number of total SGLD steps was set to $J=500$ in all cases, so significantly higher than during training. For all of our synthetic datasets and models, we generated 50 conditional samples and then formed subsets containing the $n_{E}=25$ lowest-energy samples. While in practice it would be sufficient to do this once for each model and dataset, we have chosen to perform sampling separately for each individual counterfactual in our experiments to account for stochasticity. To help reduce the computational burden for our real-world datasets we have generated only 10 conditional samples each time and used all of them in our counterfactual search. Using more samples, as we originally did, had no substantial impact on our results.
+
+\subsection{Conformal Prediction}\label{app:cp}
+
+In this Appendix~\ref{app:cp} we provide some more background on CP and explain in some more detail how we have used recent advances in Conformal Training for our purposes.
+
+\subsubsection{Background on CP}
+
+Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the data. It can be used to generate prediction intervals for regression models and prediction sets for classification models. Since the literature on CE and AR is typically concerned with classification problems, we focus on the latter. A particular variant of CP called Split Conformal Prediction (SCP) is well-suited for our purposes, because it imposes only minimal restrictions on model training.
+
+Specifically, SCP involves splitting the data $\mathcal{D}_n=\{(\mathbf{x}_i,\mathbf{y}_i)\}_{i=1,...,n}$ into a proper training set $\mathcal{D}_{\text{train}}$ and a calibration set $\mathcal{D}_{\text{cal}}$. The former is used to train the classifier in any conventional fashion. The latter is then used to compute so-called nonconformity scores: $\mathcal{S}=\{s(\mathbf{x}_i,\mathbf{y}_i)\}_{i \in \mathcal{D}_{\text{cal}}}$ where $s: (\mathcal{X},\mathcal{Y}) \mapsto \mathbb{R}$ is referred to as \textit{score function}. In the context of classification, a common choice for the score function is just $s_i=1-M_{\theta}(\mathbf{x}_i)[\mathbf{y}_i]$, that is one minus the softmax output corresponding to the observed label $\mathbf{y}_i$~\citep{angelopoulos2021gentle}.
+
+Finally, classification sets are formed as follows,
+
+\begin{equation}\label{eq:scp}
+ \begin{aligned}
+ C_{\theta}(\mathbf{x}_i;\alpha)=\{\mathbf{y}: s(\mathbf{x}_i,\mathbf{y}) \le \hat{q}\}
+ \end{aligned}
+\end{equation}
+
+where $\hat{q}$ denotes the $(1-\alpha)$-quantile of $\mathcal{S}$ and $\alpha$ is a predetermined error rate. As the size of the calibration set increases, the probability that the classification set $C(\mathbf{x}_{\text{test}})$ for a newly arrived sample $\mathbf{x}_{\text{test}}$ does not cover the true test label $\mathbf{y}_{\text{test}}$ approaches $\alpha$~\citep{angelopoulos2021gentle}.
+
+Observe from Equation~\ref{eq:scp} that Conformal Prediction works on an instance-level basis, much like CE are local. The prediction set for an individual instance $\mathbf{x}_i$ depends only on the characteristics of that sample and the specified error rate. Intuitively, the set is more likely to include multiple labels for samples that are difficult to classify, so the set size is indicative of predictive uncertainty. To see why this effect is exacerbated by small choices for $\alpha$ consider the case of $\alpha=0$, which requires that the true label is covered by the prediction set with probability equal to 1.
+
+\subsubsection{Differentiability}
+
+The fact that conformal classifiers produce set-valued predictions introduces a challenge: it is not immediately obvious how to use such classifiers in the context of gradient-based counterfactual search. Put differently, it is not clear how to use prediction sets in Equation~\ref{eq:general}. Fortunately, \citet{stutz2022learning} have recently proposed a framework for Conformal Training that also hinges on differentiability. Specifically, they show how Stochastic Gradient Descent can be used to train classifiers not only for the discriminative task but also for additional objectives related to Conformal Prediction. One such objective is \textit{efficiency}: for a given target error rate $\alpha$, the efficiency of a conformal classifier improves as its average prediction set size decreases. To this end, the authors introduce a smooth set size penalty defined in Equation~\ref{eq:setsize} in the body of this paper. Formally, it is defined as $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha):=\sigma\left((s(\mathbf{x}_i,\mathbf{y})-\alpha) T^{-1}\right)$ for $\mathbf{y}\in\mathcal{Y}$, where $\sigma$ is the sigmoid function and $T$ is a hyper-parameter used for temperature scaling~\citep{stutz2022learning}.
+
+In addition to the smooth set size penalty,~\citet{stutz2022learning} also propose a configurable classification loss function, that can be used to enforce coverage. For \textit{MNIST} data, we found that using this function generally improved the visual quality of the generated counterfactuals, so we used it in our experiments involving real-world data. For the synthetic dataset, visual inspection of the counterfactuals showed that using the configurable loss function sometimes led to overshooting: counterfactuals would end up deep inside the target domain but far away from the observed samples. For this reason, we instead relied on standard cross-entropy loss for our synthetic datasets. As we have noted in the body of the paper, more experimental work is certainly needed in this context. Figure~\ref{fig:cp-diff} shows the prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a \textit{JEM} trained on our \textit{Linearly Separable} data.
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../artifacts/results/images/poc_set_size.png}
+ \caption{Prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a JEM trained on our \textit{Linearly Separable} data.}\label{fig:cp-diff}
+\end{figure}
+
+\subsection{ECCCo}\label{app:eccco}
+
+In this section, we briefly discuss convergence conditions for CE and provide details concerning the actual implementation of our framework in \texttt{Julia}.
+\subsubsection{A Note on Convergence}
+
+Convergence is not typically discussed much in the context of CE, even though it has important implications on outcomes. One intuitive way to specify convergence is in terms of threshold probabilities: once the predicted probability $p(\mathbf{y}^+|\mathbf{x}^{\prime})$ exceeds some user-defined threshold $\gamma$ such that the counterfactual is valid, we could consider the search to have converged. In the binary case, for example, convergence could be defined as $p(\mathbf{y}^+|\mathbf{x}^{\prime})>0.5$ in this sense. Note, however, how this can be expected to yield counterfactuals in the proximity of the decision boundary, a region characterized by high aleatoric uncertainty. In other words, counterfactuals generated in this way would generally not be plausible. To avoid this from happening, we specify convergence in terms of gradients approaching zero for all our experiments and all of our generators. This is allows us to get a cleaner read on how the different counterfactual search objectives affect counterfactual outcomes.
+
+\subsubsection{\texttt{ECCCo.jl}}
+
+The core part of our code base is integrated into a larger ecosystem of \texttt{Julia} packages that we are actively developing and maintaining. To avoid compromising the double-blind review process, we only provide a link to an anonymized repository at this stage: \url{https://anonymous.4open.science/r/ECCCo-1252/README.md}.
+
+\subsection{Experimental Setup}\label{app:setup}
+
+Table~\ref{tab:params} provides an overview of all parameters related to our experiments. The \textit{GMSC} data were randomly undersampled for balancing purposes and all features were standardized. \textit{MNIST} data was also randomly undersampled for reasons outlined below. Pixel values were preprocessed to fall in the range of $[-1,1]$ and a small Gaussian noise component ($\sigma=0.03$) was added to training samples following common practice in the EBM literature. All of our models were trained through mini-batch training using the Adam optimiser (\citet{kingma2017adam}). Table~\ref{tab:perf} shows standard evaluation metrics measuring the predictive performance of our different models grouped by dataset. These measures were computed on test data.
+
+Table~\ref{tab:genparams} summarises our hyperparameter choices for the counterfactual generators where $\eta$ denotes the learning rate used for Stochastic Gradient Descent (SGD) and $\lambda_1$, $\lambda_2$, $\lambda_3$ represent the chosen penalty strengths (Equations~\ref{eq:general} and~\ref{eq:eccco}). Here $\lambda_1$ also refers to the chosen penalty for the distance from factual values that applies to both \textit{Wachter} and \textit{REVISE}, but not \textit{Schut} which is penalty-free. \textit{Schut} is also the only generator that uses JSMA instead of SGD for optimization.
+
+\import{contents/}{table_params.tex}
+
+\import{contents/}{table_perf.tex}
+
+\import{contents/}{table_gen_params.tex}
+
+\subsubsection{Compute}
+
+To enable others to easily replicate our experiments, we have chosen to work with small neural network architectures and randomly undersampled the \textit{MNIST} dataset (maintaining class balance). All of our experiments could then be run locally on a personal machine. The longest runtimes we experienced for model training and counterfactual benchmarking were on the order of 8-12 hours (\textit{MNIST} data). For the synthetic data, all experiments could be completed in less than an hour.
+
+We have summarised our system information below:
+
+\textbf{Software}:
+
+\begin{itemize}
+ \item System Version: macOS 13.3.1
+ \item Kernel Version: Darwin 22.4.0
+\end{itemize}
+
+\textbf{Hardware}:
+
+\begin{itemize}
+ \item Model Name: MacBook Pro
+ \item Model Identifier: MacBookPro16,1
+ \item Processor Name: 8-Core Intel Core i9
+ \item Processor Speed: 2.3 GHz
+ \item Number of Processors: 1
+ \item Total Number of Cores: 8
+ \item L2 Cache (per Core): 256 KB
+ \item L3 Cache: 16 MB
+ \item Hyper-Threading Technology: Enabled
+ \item Memory: 32 GB
+\end{itemize}
+
+
+\subsection{Results}\label{app:results}
+
+Figure~\ref{fig:mnist-eccco} shows examples of counterfactuals for \textit{MNIST} data where the underlying model is our \textit{JEM Ensemble}. Original images are shown on the diagonal and the corresponding counterfactuals are plotted across rows.
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.9\linewidth]{../artifacts/results/images/mnist_eccco_all_digits.png}
+ \caption{Counterfactuals for \textit{MNIST} data and our \textit{JEM Ensemble}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco}
+\end{figure}
+
+Table~\ref{tab:results-full} reports all of the evaluation metrics we have computed. Table~\ref{tab:results-full-valid} reports the same metrics for the subset of valid counterfactuals. The `Unfaithfulness' and `Implausibility' metrics have been discussed extensively in the body of the paper. The `Cost' metric relates to the distance between the factual and the counterfactual. The `Redundancy' metric measures sparsity in is defined as the percentage of features that remain unperturbed (higher is better). The `Uncertainty' metric is just the average value of the smooth set size penalty (Equation~\ref{eq:setsize}). Finally, `Validity' is the percentage of valid counterfactuals.
+
+\import{contents/}{table_all.tex}
+
+\import{contents/}{table_all_valid.tex}
\ No newline at end of file
diff --git a/paper/paper.tex b/paper/body.tex
similarity index 65%
rename from paper/paper.tex
rename to paper/body.tex
index 58c9610bf326e89cf055b136131457967c80834f..1437df3311215e75f16914e6585b75ec82629b68 100644
--- a/paper/paper.tex
+++ b/paper/body.tex
@@ -1,91 +1,3 @@
-\documentclass{article}
-
-
-% if you need to pass options to natbib, use, e.g.:
-% \PassOptionsToPackage{numbers, compress}{natbib}
-% before loading neurips_2023
-
-
-% ready for submission
-\usepackage{neurips_2023}
-
-
-% to compile a preprint version, e.g., for submission to arXiv, add add the
-% [preprint] option:
-% \usepackage[preprint]{neurips_2023}
-
-
-% to compile a camera-ready version, add the [final] option, e.g.:
-% \usepackage[final]{neurips_2023}
-
-
-% to avoid loading the natbib package, add option nonatbib:
-% \usepackage[nonatbib]{neurips_2023}
-
-
-\usepackage[utf8]{inputenc} % allow utf-8 input
-\usepackage[T1]{fontenc} % use 8-bit T1 fonts
-\usepackage{hyperref} % hyperlinks
-\usepackage{url} % simple URL typesetting
-\usepackage{booktabs} % professional-quality tables
-\usepackage{amsfonts} % blackboard math symbols
-\usepackage{nicefrac} % compact symbols for 1/2, etc.
-\usepackage{microtype} % microtypography
-\usepackage{xcolor} % colors
-
-\usepackage{amsmath}
-\usepackage{amsthm}
-\usepackage{caption}
-\usepackage{graphicx}
-\usepackage{algorithm}
-\usepackage{algpseudocode}
-\usepackage{import}
-\usepackage{booktabs}
-\usepackage{longtable}
-\usepackage{array}
-\usepackage{multirow}
-
-% Bibliography
-\bibliographystyle{unsrtnat}
-\setcitestyle{numbers,square,comma}
-
-% Numbered Environments:
-\newtheorem{definition}{Definition}[section]
-\newtheorem{question}{Research Question}[section]
-
-
-\title{ECCCos from the Black Box:\\
-Faithful Explanations through\\
-Energy-Constrained Conformal Counterfactuals}
-
-
-% The \author macro works with any number of authors. There are two commands
-% used to separate the names and addresses of multiple authors: \And and \AND.
-%
-% Using \And between authors leaves it to LaTeX to determine where to break the
-% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
-% authors names on the first line, and the last on the second line, try using
-% \AND instead of \And before the third author name.
-
-
-\author{%
- Anonymous Author\thanks{See also: } \\
- Faculty \\
- University \\
- Address \\
- \texttt{email} \\
- \And
- Anonymous Author\thanks{See also: } \\
- Faculty \\
- University \\
- Address \\
- \texttt{email} \\
-}
-
-
-\begin{document}
-
-
\maketitle
@@ -144,7 +56,7 @@ To demonstrate this with an example, we have trained a simple image classifier $
\begin{figure}
\centering
- \includegraphics[width=0.8\textwidth]{../artifacts/results/images/mnist_motivation.png}
+ \includegraphics[width=0.8\linewidth]{../artifacts/results/images/mnist_motivation.png}
\caption{Counterfactuals for turning a 9 (nine) into a 7 (seven): original image (left); then from left to right the counterfactuals generated using \textit{Wachter}, \textit{Schut} and \textit{REVISE}.}\label{fig:motiv}
\end{figure}
@@ -228,13 +140,10 @@ Here, $\kappa \in \{0,1\}$ is a hyper-parameter and $C_{\theta,\mathbf{y}}(\math
\begin{figure}
\centering
- \includegraphics[width=1.0\textwidth]{../artifacts/results/images/poc_gradient_fields.png}
+ \includegraphics[width=1.0\linewidth]{../artifacts/results/images/poc_gradient_fields.png}
\caption{Gradient fields and counterfactual paths for different generators. The objective is to generate a counterfual in the `blue' class for a sample from the `orange' class. Bright yellow stars indicate conditional samples generated through SGLD. The underlying classifier is a Joint Energy Model.}\label{fig:poc}
\end{figure}
-\renewcommand{\algorithmicrequire}{\textbf{Input:}}
-\renewcommand{\algorithmicensure}{\textbf{Output:}}
-
\begin{algorithm}
\caption{The \textit{ECCCo} generator}\label{alg:eccco}
\begin{algorithmic}[1]
@@ -317,175 +226,4 @@ While our approach is readily applicable to models with gradient access like dee
\section{Conclusion}
-This work leverages recent advances in Energy-Based Modelling and Conformal Prediction in the context of Explainable Artificial Intelligence. We have proposed a new way to generate counterfactuals that are maximally faithful to the black-box model they aim to explain. Our proposed generator, \textit{ECCCo}, produces plausible counterfactuals if and only if the black-box model itself has learned realistic explanations for the data, which we have demonstrated through rigorous empirical analysis. This should enable researchers and practitioners to use counterfactuals in order to discern trustworthy models from unreliable ones. While the scope of this work limits its generalizability, we believe that \textit{ECCCo} offers a solid baseline for future work on faithful Counterfactual Explanations.
-
-\begin{ack}
-
-Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
-collaboration.
-
-\end{ack}
-
-\bibliography{bib}
-
-\pagebreak
-
-\appendix
-\section*{Appendices}
-\renewcommand{\thesubsection}{\Alph{subsection}}
-
-The following appendices provide additional details that are relevant to the paper. Appendices~\ref{app:jem} and~\ref{app:cp} explain any tasks related to Energy-Based Modelling and Predictive Uncertainty Quantification through Conformal Prediction, respectively. Appendix~\ref{app:eccco} provides additional technical and implementation details about our proposed generator, \textit{ECCCo}, including references to our open-sourced code base. A complete overview of our experimental setup detailing our parameter choices, training procedures and initial black-box model performance can be found in Appendix~\ref{app:setup}. Finally, Appendix~\ref{app:results} reports all of our experimental results in more detail.
-
-\subsection{Energy-Based Modelling}\label{app:jem}
-
-Since we were not able to identify any existing open-source software for Energy-Based Modelling that would be flexible enough to cater to our needs, we have developed a \texttt{Julia} package from scratch. The package has been open-sourced, but to avoid compromising the double-blind review process, we refrain from providing more information at this stage. In our development we have heavily drawn on the existing literature:~\citet{du2020implicit} describe best practices for using EBM for generative modelling;~\citet{grathwohl2020your} explain how EBM can be used to train classifiers jointly for the discriminative and generative tasks. We have used the same package for training and inference, but there are some important differences between the two cases that are worth highlighting here.
-
-\subsubsection{Training: Joint Energy Models}
-
-To train our Joint Energy Models we broadly follow the approach outlined in~\citet{grathwohl2020your}. These models are trained to optimize a hybrid objective that involves a standard classification loss component $L_{\text{clf}}(\theta)=-\log p_{\theta}(\mathbf{y}|\mathbf{x})$ (e.g. cross-entropy loss) as well as a generative loss component $L_{\text{gen}}(\theta)=-\log p_{\theta}(\mathbf{x})$.
-
-To draw samples from $p_{\theta}(\mathbf{x})$, we rely exclusively on the conditional sampling approach described in~\citet{grathwohl2020your} for both training and inference: we first draw $\mathbf{y}\sim p(\mathbf{y})$ and then sample $\mathbf{x} \sim p_{\theta}(\mathbf{x}|\mathbf{y})$~\citep{grathwohl2020your} via Equation~\ref{eq:sgld} with energy $\mathcal{E}(\mathbf{x}|\mathbf{y})=\mu_{\theta}(\mathbf{x})[\mathbf{y}]$ where $\mu_{\theta}: \mathcal{X} \mapsto \mathbb{R}^K$ returns the linear predictions (logits) of our classifier $M_{\theta}$. While our package also supports unconditional sampling, we found conditional sampling to work well. It is also well aligned with CE, since in this context we are interested in conditioning on the target class.
-
-As mentioned in the body of the paper, we rely on a biased sampler involving separately specified values for the step size $\epsilon$ and the standard deviation $\sigma$ of the stochastic term involving $\mathbf{r}$. Formally, our biased sampler performs updates as follows:
-
-\begin{equation}\label{eq:biased-sgld}
- \begin{aligned}
- \hat{\mathbf{x}}_{j+1} &\leftarrow \hat{\mathbf{x}}_j - \frac{\epsilon}{2} \mathcal{E}(\hat{\mathbf{x}}_j|\mathbf{y}^+) + \sigma \mathbf{r}_j, && j=1,...,J
- \end{aligned}
-\end{equation}
-
-Consistent with~\citet{grathwohl2020your}, we have specified $\epsilon=2$ and $\sigma=0.01$ as the default values for all of our experiments. The number of total SGLD steps $J$ varies by dataset (Table~\ref{tab:ebmparams}). Following best practices, we initialize $\mathbf{x}_0$ randomly in 5\% of all cases and sample from a buffer in all other cases. The buffer itself is randomly initialised and gradually grows to a maximum of 10,000 samples during training as $\hat{\mathbf{x}}_{J}$ is stored in each epoch~\citep{du2020implicit,grathwohl2020your}.
-
-It is important to realise that sampling is done during each training epoch, which makes training Joint Energy Models significantly harder than conventional neural classifiers. In each epoch the generated (batch of) sample(s) $\hat{\mathbf{x}}_{J}$ is used as part of the generative loss component, which compares its energy to that of observed samples $\mathbf{x}$: $L_{\text{gen}}(\theta)=\mu_{\theta}(\mathbf{x})[\mathbf{y}]-\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]$. Our full training objective can be summarized as follows,
-
-\begin{equation}\label{eq:jem-loss}
- \begin{aligned}
- L(\theta) &= L_{\text{clf}}(\theta) + L_{\text{gen}}(\theta) + \lambda L_{\text{reg}}(\theta)
- \end{aligned}
-\end{equation}
-
-where $L_{\text{reg}}(\theta)$ is a Ridge penalty (L2 norm) that regularises energy magnitudes for both observed and generated samples~\citep{du2020implicit}. We have used varying degrees of regularization depending on the dataset ($\lambda$ in Table~\ref{tab:ebmparams}).
-
-Contrary to existing work, we have not typically used the entire minibatch of training data for the generative loss component but found that using a subset of the minibatch was often sufficient in attaining decent generative performance (Table~\ref{tab:ebmparams}). This has helped to reduce the computational burden for our models, which should make it easier for others to reproduce our findings. Figures~\ref{fig:mnist-gen} and~\ref{fig:moons-gen} show generated samples for our \textit{MNIST} and \textit{Moons} data, to provide a sense of their generative property.
-
-\import{contents/}{table_ebm_params.tex}
-
-\begin{figure}
- \centering
- \includegraphics[width=0.75\textwidth]{../artifacts/results/images/mnist_generated_JEM Ensemble.png}
- \caption{Conditionally generated \textit{MNIST} images for our JEM Ensemble.}\label{fig:mnist-gen}
-\end{figure}
-
-\begin{figure}
- \centering
- \includegraphics[width=0.5\textwidth]{../artifacts/results/images/moons_generated_JEM.png}
- \caption{Conditionally generated samples (stars) for our \textit{Moons} data using a JEM.}\label{fig:moons-gen}
-\end{figure}
-\subsubsection{Inference: Quantifying Models' Generative Property}
-
-At inference time, we assume no prior knowledge about the model's generative property. This means that we do not tab into the existing buffer of generated samples for our Joint Energy Models, but instead generate conditional samples from scratch. While we have relied on the default values $\epsilon=2$ and $\sigma=0.01$ also during inference, the number of total SGLD steps was set to $J=500$ in all cases, so significantly higher than during training. For all of our synthetic datasets and models, we generated 50 conditional samples and then formed subsets containing the $n_{E}=25$ lowest-energy samples. While in practice it would be sufficient to do this once for each model and dataset, we have chosen to perform sampling separately for each individual counterfactual in our experiments to account for stochasticity. To help reduce the computational burden for our real-world datasets we have generated only 10 conditional samples each time and used all of them in our counterfactual search. Using more samples, as we originally did, had no substantial impact on our results.
-
-\subsection{Conformal Prediction}\label{app:cp}
-
-In this Appendix~\ref{app:cp} we provide some more background on CP and explain in some more detail how we have used recent advances in Conformal Training for our purposes.
-
-\subsubsection{Background on CP}
-
-Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the data. It can be used to generate prediction intervals for regression models and prediction sets for classification models. Since the literature on CE and AR is typically concerned with classification problems, we focus on the latter. A particular variant of CP called Split Conformal Prediction (SCP) is well-suited for our purposes, because it imposes only minimal restrictions on model training.
-
-Specifically, SCP involves splitting the data $\mathcal{D}_n=\{(\mathbf{x}_i,\mathbf{y}_i)\}_{i=1,...,n}$ into a proper training set $\mathcal{D}_{\text{train}}$ and a calibration set $\mathcal{D}_{\text{cal}}$. The former is used to train the classifier in any conventional fashion. The latter is then used to compute so-called nonconformity scores: $\mathcal{S}=\{s(\mathbf{x}_i,\mathbf{y}_i)\}_{i \in \mathcal{D}_{\text{cal}}}$ where $s: (\mathcal{X},\mathcal{Y}) \mapsto \mathbb{R}$ is referred to as \textit{score function}. In the context of classification, a common choice for the score function is just $s_i=1-M_{\theta}(\mathbf{x}_i)[\mathbf{y}_i]$, that is one minus the softmax output corresponding to the observed label $\mathbf{y}_i$~\citep{angelopoulos2021gentle}.
-
-Finally, classification sets are formed as follows,
-
-\begin{equation}\label{eq:scp}
- \begin{aligned}
- C_{\theta}(\mathbf{x}_i;\alpha)=\{\mathbf{y}: s(\mathbf{x}_i,\mathbf{y}) \le \hat{q}\}
- \end{aligned}
-\end{equation}
-
-where $\hat{q}$ denotes the $(1-\alpha)$-quantile of $\mathcal{S}$ and $\alpha$ is a predetermined error rate. As the size of the calibration set increases, the probability that the classification set $C(\mathbf{x}_{\text{test}})$ for a newly arrived sample $\mathbf{x}_{\text{test}}$ does not cover the true test label $\mathbf{y}_{\text{test}}$ approaches $\alpha$~\citep{angelopoulos2021gentle}.
-
-Observe from Equation~\ref{eq:scp} that Conformal Prediction works on an instance-level basis, much like CE are local. The prediction set for an individual instance $\mathbf{x}_i$ depends only on the characteristics of that sample and the specified error rate. Intuitively, the set is more likely to include multiple labels for samples that are difficult to classify, so the set size is indicative of predictive uncertainty. To see why this effect is exacerbated by small choices for $\alpha$ consider the case of $\alpha=0$, which requires that the true label is covered by the prediction set with probability equal to 1.
-
-\subsubsection{Differentiability}
-
-The fact that conformal classifiers produce set-valued predictions introduces a challenge: it is not immediately obvious how to use such classifiers in the context of gradient-based counterfactual search. Put differently, it is not clear how to use prediction sets in Equation~\ref{eq:general}. Fortunately, \citet{stutz2022learning} have recently proposed a framework for Conformal Training that also hinges on differentiability. Specifically, they show how Stochastic Gradient Descent can be used to train classifiers not only for the discriminative task but also for additional objectives related to Conformal Prediction. One such objective is \textit{efficiency}: for a given target error rate $\alpha$, the efficiency of a conformal classifier improves as its average prediction set size decreases. To this end, the authors introduce a smooth set size penalty defined in Equation~\ref{eq:setsize} in the body of this paper. Formally, it is defined as $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha):=\sigma\left((s(\mathbf{x}_i,\mathbf{y})-\alpha) T^{-1}\right)$ for $\mathbf{y}\in\mathcal{Y}$, where $\sigma$ is the sigmoid function and $T$ is a hyper-parameter used for temperature scaling~\citep{stutz2022learning}.
-
-In addition to the smooth set size penalty,~\citet{stutz2022learning} also propose a configurable classification loss function, that can be used to enforce coverage. For \textit{MNIST} data, we found that using this function generally improved the visual quality of the generated counterfactuals, so we used it in our experiments involving real-world data. For the synthetic dataset, visual inspection of the counterfactuals showed that using the configurable loss function sometimes led to overshooting: counterfactuals would end up deep inside the target domain but far away from the observed samples. For this reason, we instead relied on standard cross-entropy loss for our synthetic datasets. As we have noted in the body of the paper, more experimental work is certainly needed in this context. Figure~\ref{fig:cp-diff} shows the prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a \textit{JEM} trained on our \textit{Linearly Separable} data.
-
-\begin{figure}
- \centering
- \includegraphics[width=1.0\textwidth]{../artifacts/results/images/poc_set_size.png}
- \caption{Prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a JEM trained on our \textit{Linearly Separable} data.}\label{fig:cp-diff}
-\end{figure}
-
-\subsection{ECCCo}\label{app:eccco}
-
-In this section, we briefly discuss convergence conditions for CE and provide details concerning the actual implementation of our framework in \texttt{Julia}.
-\subsubsection{A Note on Convergence}
-
-Convergence is not typically discussed much in the context of CE, even though it has important implications on outcomes. One intuitive way to specify convergence is in terms of threshold probabilities: once the predicted probability $p(\mathbf{y}^+|\mathbf{x}^{\prime})$ exceeds some user-defined threshold $\gamma$ such that the counterfactual is valid, we could consider the search to have converged. In the binary case, for example, convergence could be defined as $p(\mathbf{y}^+|\mathbf{x}^{\prime})>0.5$ in this sense. Note, however, how this can be expected to yield counterfactuals in the proximity of the decision boundary, a region characterized by high aleatoric uncertainty. In other words, counterfactuals generated in this way would generally not be plausible. To avoid this from happening, we specify convergence in terms of gradients approaching zero for all our experiments and all of our generators. This is allows us to get a cleaner read on how the different counterfactual search objectives affect counterfactual outcomes.
-
-\subsubsection{\texttt{ECCCo.jl}}
-
-The core part of our code base is integrated into a larger ecosystem of \texttt{Julia} packages that we are actively developing and maintaining. To avoid compromising the double-blind review process, we only provide a link to an anonymized repository at this stage: \url{https://anonymous.4open.science/r/ECCCo-1252/README.md}.
-
-\subsection{Experimental Setup}\label{app:setup}
-
-Table~\ref{tab:params} provides an overview of all parameters related to our experiments. The \textit{GMSC} data were randomly undersampled for balancing purposes and all features were standardized. \textit{MNIST} data was also randomly undersampled for reasons outlined below. Pixel values were preprocessed to fall in the range of $[-1,1]$ and a small Gaussian noise component ($\sigma=0.03$) was added to training samples following common practice in the EBM literature. All of our models were trained through mini-batch training using the Adam optimiser (\citet{kingma2017adam}). Table~\ref{tab:perf} shows standard evaluation metrics measuring the predictive performance of our different models grouped by dataset. These measures were computed on test data.
-
-Table~\ref{tab:genparams} summarises our hyperparameter choices for the counterfactual generators where $\eta$ denotes the learning rate used for Stochastic Gradient Descent (SGD) and $\lambda_1$, $\lambda_2$, $\lambda_3$ represent the chosen penalty strengths (Equations~\ref{eq:general} and~\ref{eq:eccco}). Here $\lambda_1$ also refers to the chosen penalty for the distance from factual values that applies to both \textit{Wachter} and \textit{REVISE}, but not \textit{Schut} which is penalty-free. \textit{Schut} is also the only generator that uses JSMA instead of SGD for optimization.
-
-\import{contents/}{table_params.tex}
-
-\import{contents/}{table_perf.tex}
-
-\import{contents/}{table_gen_params.tex}
-
-\subsubsection{Compute}
-
-To enable others to easily replicate our experiments, we have chosen to work with small neural network architectures and randomly undersampled the \textit{MNIST} dataset (maintaining class balance). All of our experiments could then be run locally on a personal machine. The longest runtimes we experienced for model training and counterfactual benchmarking were on the order of 8-12 hours (\textit{MNIST} data). For the synthetic data, all experiments could be completed in less than an hour.
-
-We have summarised our system information below:
-
-\textbf{Software}:
-
-\begin{itemize}
- \item System Version: macOS 13.3.1
- \item Kernel Version: Darwin 22.4.0
-\end{itemize}
-
-\textbf{Hardware}:
-
-\begin{itemize}
- \item Model Name: MacBook Pro
- \item Model Identifier: MacBookPro16,1
- \item Processor Name: 8-Core Intel Core i9
- \item Processor Speed: 2.3 GHz
- \item Number of Processors: 1
- \item Total Number of Cores: 8
- \item L2 Cache (per Core): 256 KB
- \item L3 Cache: 16 MB
- \item Hyper-Threading Technology: Enabled
- \item Memory: 32 GB
-\end{itemize}
-
-
-\subsection{Results}\label{app:results}
-
-Figure~\ref{fig:mnist-eccco} shows examples of counterfactuals for \textit{MNIST} data where the underlying model is our \textit{JEM Ensemble}. Original images are shown on the diagonal and the corresponding counterfactuals are plotted across rows.
-
-\begin{figure}
- \centering
- \includegraphics[width=0.9\textwidth]{../artifacts/results/images/mnist_eccco_all_digits.png}
- \caption{Counterfactuals for \textit{MNIST} data and our \textit{JEM Ensemble}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco}
-\end{figure}
-
-Table~\ref{tab:results-full} reports all of the evaluation metrics we have computed. Table~\ref{tab:results-full-valid} reports the same metrics for the subset of valid counterfactuals. The `Unfaithfulness' and `Implausibility' metrics have been discussed extensively in the body of the paper. The `Cost' metric relates to the distance between the factual and the counterfactual. The `Redundancy' metric measures sparsity in is defined as the percentage of features that remain unperturbed (higher is better). The `Uncertainty' metric is just the average value of the smooth set size penalty (Equation~\ref{eq:setsize}). Finally, `Validity' is the percentage of valid counterfactuals.
-
-\import{contents/}{table_all.tex}
-
-\import{contents/}{table_all_valid.tex}
-
-
-\end{document}
\ No newline at end of file
+This work leverages recent advances in Energy-Based Modelling and Conformal Prediction in the context of Explainable Artificial Intelligence. We have proposed a new way to generate counterfactuals that are maximally faithful to the black-box model they aim to explain. Our proposed generator, \textit{ECCCo}, produces plausible counterfactuals if and only if the black-box model itself has learned realistic explanations for the data, which we have demonstrated through rigorous empirical analysis. This should enable researchers and practitioners to use counterfactuals in order to discern trustworthy models from unreliable ones. While the scope of this work limits its generalizability, we believe that \textit{ECCCo} offers a solid baseline for future work on faithful Counterfactual Explanations.
\ No newline at end of file
diff --git a/paper/neurips_2023.sty b/paper/neurips/neurips_2023.sty
similarity index 100%
rename from paper/neurips_2023.sty
rename to paper/neurips/neurips_2023.sty
diff --git a/paper/paper.pdf b/paper/neurips/paper.pdf
similarity index 99%
rename from paper/paper.pdf
rename to paper/neurips/paper.pdf
index 6a465936f6699620fc527e6085cfbfc0d5a1467e..bb9b3f4c9561675ccb1efddc627b82c5baf403f5 100644
Binary files a/paper/paper.pdf and b/paper/neurips/paper.pdf differ
diff --git a/paper/neurips/paper.tex b/paper/neurips/paper.tex
new file mode 100644
index 0000000000000000000000000000000000000000..958928cff8e8e376e5d2c98a5c3943f9a3197db5
--- /dev/null
+++ b/paper/neurips/paper.tex
@@ -0,0 +1,105 @@
+\documentclass{article}
+
+
+% if you need to pass options to natbib, use, e.g.:
+% \PassOptionsToPackage{numbers, compress}{natbib}
+% before loading neurips_2023
+
+
+% ready for submission
+\usepackage{neurips_2023}
+
+
+% to compile a preprint version, e.g., for submission to arXiv, add add the
+% [preprint] option:
+% \usepackage[preprint]{neurips_2023}
+
+
+% to compile a camera-ready version, add the [final] option, e.g.:
+% \usepackage[final]{neurips_2023}
+
+
+% to avoid loading the natbib package, add option nonatbib:
+% \usepackage[nonatbib]{neurips_2023}
+
+
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc} % use 8-bit T1 fonts
+\usepackage{hyperref} % hyperlinks
+\usepackage{url} % simple URL typesetting
+\usepackage{booktabs} % professional-quality tables
+\usepackage{amsfonts} % blackboard math symbols
+\usepackage{nicefrac} % compact symbols for 1/2, etc.
+\usepackage{microtype} % microtypography
+\usepackage{xcolor} % colors
+
+\usepackage{amsmath}
+\usepackage{amsthm}
+\usepackage{caption}
+\usepackage{graphicx}
+\usepackage{algorithm}
+\usepackage{algpseudocode}
+\usepackage{import}
+\usepackage{booktabs}
+\usepackage{longtable}
+\usepackage{array}
+\usepackage{multirow}
+
+% Bibliography
+\bibliographystyle{unsrtnat}
+\setcitestyle{numbers,square,comma}
+
+% Numbered Environments:
+\newtheorem{definition}{Definition}[section]
+\newtheorem{question}{Research Question}[section]
+
+\renewcommand{\algorithmicrequire}{\textbf{Input:}}
+\renewcommand{\algorithmicensure}{\textbf{Output:}}
+
+
+\title{ECCCos from the Black Box:\\
+Faithful Explanations through\\
+Energy-Constrained Conformal Counterfactuals}
+
+
+% The \author macro works with any number of authors. There are two commands
+% used to separate the names and addresses of multiple authors: \And and \AND.
+%
+% Using \And between authors leaves it to LaTeX to determine where to break the
+% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
+% authors names on the first line, and the last on the second line, try using
+% \AND instead of \And before the third author name.
+
+
+\author{%
+ Anonymous Author\thanks{See also: } \\
+ Faculty \\
+ University \\
+ Address \\
+ \texttt{email} \\
+ \And
+ Anonymous Author\thanks{See also: } \\
+ Faculty \\
+ University \\
+ Address \\
+ \texttt{email} \\
+}
+
+
+\begin{document}
+
+% Body of the paper
+\import{../}{body.tex}
+
+\begin{ack}
+
+Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
+collaboration.
+
+\end{ack}
+
+\bibliography{../bib}
+
+\import{../}{appendix.tex}
+
+\end{document}
\ No newline at end of file