diff --git a/.gitignore b/.gitignore
index 3e62d6da5c06bc75a0b44541b4d6a2ba62432577..9177922f448795d64bb71647b4d3d38be22b6c6a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
# Other:
/artifacts/
-/.quarto/
+**/.quarto/
/Manifest.toml
/results*/
**/.CondaPkg
diff --git a/bib.bib b/bib.bib
deleted file mode 100644
index daed3cba558bf28a0516e5454625b5b48ea016e8..0000000000000000000000000000000000000000
--- a/bib.bib
+++ /dev/null
@@ -1,2837 +0,0 @@
-@TechReport{kingma2017adam,
- author = {Kingma, Diederik P. and Ba, Jimmy},
- date = {2017-01},
- institution = {arXiv},
- title = {Adam: {A} {Method} for {Stochastic} {Optimization}},
- doi = {10.48550/arXiv.1412.6980},
- note = {arXiv:1412.6980 [cs] type: article},
- url = {http://arxiv.org/abs/1412.6980},
- urldate = {2023-05-17},
- abstract = {We introduce Adam, an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. The method is straightforward to implement, is computationally efficient, has little memory requirements, is invariant to diagonal rescaling of the gradients, and is well suited for problems that are large in terms of data and/or parameters. The method is also appropriate for non-stationary objectives and problems with very noisy and/or sparse gradients. The hyper-parameters have intuitive interpretations and typically require little tuning. Some connections to related algorithms, on which Adam was inspired, are discussed. We also analyze the theoretical convergence properties of the algorithm and provide a regret bound on the convergence rate that is comparable to the best known results under the online convex optimization framework. Empirical results demonstrate that Adam works well in practice and compares favorably to other stochastic optimization methods. Finally, we discuss AdaMax, a variant of Adam based on the infinity norm.},
- annotation = {Comment: Published as a conference paper at the 3rd International Conference for Learning Representations, San Diego, 2015},
- file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1412.6980.pdf:application/pdf},
- keywords = {Computer Science - Machine Learning},
- shorttitle = {Adam},
-}
-
-@TechReport{xiao2017fashion,
- author = {Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
- date = {2017-09},
- institution = {arXiv},
- title = {Fashion-{MNIST}: a {Novel} {Image} {Dataset} for {Benchmarking} {Machine} {Learning} {Algorithms}},
- doi = {10.48550/arXiv.1708.07747},
- note = {arXiv:1708.07747 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1708.07747},
- urldate = {2023-05-10},
- abstract = {We present Fashion-MNIST, a new dataset comprising of 28x28 grayscale images of 70,000 fashion products from 10 categories, with 7,000 images per category. The training set has 60,000 images and the test set has 10,000 images. Fashion-MNIST is intended to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms, as it shares the same image size, data format and the structure of training and testing splits. The dataset is freely available at https://github.com/zalandoresearch/fashion-mnist},
- annotation = {Comment: Dataset is freely available at https://github.com/zalandoresearch/fashion-mnist Benchmark is available at http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/},
- file = {:xiao2017fashion - Fashion MNIST_ a Novel Image Dataset for Benchmarking Machine Learning Algorithms.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Statistics - Machine Learning},
- shorttitle = {Fashion-{MNIST}},
-}
-
-@Online{mw2023fidelity,
- author = {Merriam-Webster},
- title = {"Fidelity"},
- url = {https://www.merriam-webster.com/dictionary/fidelity},
- language = {en},
- organization = {Merriam-Webster},
- urldate = {2023-03-23},
- abstract = {the quality or state of being faithful; accuracy in details : exactness; the degree to which an electronic device (such as a record player, radio, or television) accurately reproduces its effect (such as sound or picture)… See the full definition},
-}
-
-@InProceedings{altmeyer2023endogenous,
- author = {Altmeyer, Patrick and Angela, Giovan and Buszydlik, Aleksander and Dobiczek, Karol and van Deursen, Arie and Liem, Cynthia},
- booktitle = {First {IEEE} {Conference} on {Secure} and {Trustworthy} {Machine} {Learning}},
- title = {Endogenous {Macrodynamics} in {Algorithmic} {Recourse}},
- file = {:altmeyerendogenous - Endogenous Macrodynamics in Algorithmic Recourse.pdf:PDF},
- year = {2023},
-}
-
-%% This BibTeX bibliography file was created using BibDesk.
-%% https://bibdesk.sourceforge.io/
-
-%% Created for Anonymous Author at 2022-12-13 12:58:22 +0100
-
-
-%% Saved with string encoding Unicode (UTF-8)
-
-
-
-@Article{abadie2002instrumental,
- author = {Abadie, Alberto and Angrist, Joshua and Imbens, Guido},
- title = {Instrumental Variables Estimates of the Effect of Subsidized Training on the Quantiles of Trainee Earnings},
- number = {1},
- pages = {91--117},
- volume = {70},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica : journal of the Econometric Society},
- shortjournal = {Econometrica},
- year = {2002},
-}
-
-@Article{abadie2003economic,
- author = {Abadie, Alberto and Gardeazabal, Javier},
- title = {The Economic Costs of Conflict: {{A}} Case Study of the {{Basque Country}}},
- number = {1},
- pages = {113--132},
- volume = {93},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {American economic review},
- year = {2003},
-}
-
-@InProceedings{ackerman2021machine,
- author = {Ackerman, Samuel and Dube, Parijat and Farchi, Eitan and Raz, Orna and Zalmanovici, Marcel},
- booktitle = {2021 {{IEEE}}/{{ACM Third International Workshop}} on {{Deep Learning}} for {{Testing}} and {{Testing}} for {{Deep Learning}} ({{DeepTest}})},
- title = {Machine {{Learning Model Drift Detection Via Weak Data Slices}}},
- pages = {1--8},
- publisher = {{IEEE}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Article{allen2017referencedependent,
- author = {Allen, Eric J and Dechow, Patricia M and Pope, Devin G and Wu, George},
- title = {Reference-Dependent Preferences: {{Evidence}} from Marathon Runners},
- number = {6},
- pages = {1657--1672},
- volume = {63},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Management Science},
- year = {2017},
-}
-
-@Article{altmeyer2018option,
- author = {Altmeyer, Patrick and Grapendal, Jacob Daniel and Pravosud, Makar and Quintana, Gand Derry},
- title = {Option Pricing in the {{Heston}} Stochastic Volatility Model: An Empirical Evaluation},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2018},
-}
-
-@Article{altmeyer2021deep,
- author = {Altmeyer, Patrick and Agusti, Marc and Vidal-Quadras Costa, Ignacio},
- title = {Deep {{Vector Autoregression}} for {{Macroeconomic Data}}},
- url = {https://thevoice.bse.eu/wp-content/uploads/2021/07/ds21-project-agusti-et-al.pdf},
- bdsk-url-1 = {https://thevoice.bse.eu/wp-content/uploads/2021/07/ds21-project-agusti-et-al.pdf},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Book{altmeyer2021deepvars,
- author = {Altmeyer, Patrick},
- title = {Deepvars: {{Deep Vector Autoregession}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Misc{altmeyer2022counterfactualexplanations,
- author = {Altmeyer, Patrick},
- title = {{{CounterfactualExplanations}}.Jl - a {{Julia}} Package for {{Counterfactual Explanations}} and {{Algorithmic Recourse}}},
- url = {https://github.com/pat-alt/CounterfactualExplanations.jl},
- bdsk-url-1 = {https://github.com/pat-alt/CounterfactualExplanations.jl},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2022},
-}
-
-@Software{altmeyerCounterfactualExplanationsJlJulia2022,
- author = {Altmeyer, Patrick},
- title = {{{CounterfactualExplanations}}.Jl - a {{Julia}} Package for {{Counterfactual Explanations}} and {{Algorithmic Recourse}}},
- url = {https://github.com/pat-alt/CounterfactualExplanations.jl},
- version = {0.1.2},
- bdsk-url-1 = {https://github.com/pat-alt/CounterfactualExplanations.jl},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2022},
-}
-
-@Unpublished{angelopoulos2021gentle,
- author = {Angelopoulos, Anastasios N. and Bates, Stephen},
- title = {A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2107.07511},
- eprinttype = {arxiv},
- file = {:/Users/FA31DU/Zotero/storage/RKSUMYZG/Angelopoulos and Bates - 2021 - A gentle introduction to conformal prediction and .pdf:;:/Users/FA31DU/Zotero/storage/PRUEKRR3/2107.html:},
- year = {2021},
-}
-
-@Misc{angelopoulos2022uncertainty,
- author = {Angelopoulos, Anastasios and Bates, Stephen and Malik, Jitendra and Jordan, Michael I.},
- title = {Uncertainty {{Sets}} for {{Image Classifiers}} Using {{Conformal Prediction}}},
- eprint = {2009.14193},
- eprinttype = {arxiv},
- abstract = {Convolutional image classifiers can achieve high predictive accuracy, but quantifying their uncertainty remains an unresolved challenge, hindering their deployment in consequential settings. Existing uncertainty quantification techniques, such as Platt scaling, attempt to calibrate the network's probability estimates, but they do not have formal guarantees. We present an algorithm that modifies any classifier to output a predictive set containing the true label with a user-specified probability, such as 90\%. The algorithm is simple and fast like Platt scaling, but provides a formal finite-sample coverage guarantee for every model and dataset. Our method modifies an existing conformal prediction algorithm to give more stable predictive sets by regularizing the small scores of unlikely classes after Platt scaling. In experiments on both Imagenet and Imagenet-V2 with ResNet-152 and other classifiers, our scheme outperforms existing approaches, achieving coverage with sets that are often factors of 5 to 10 smaller than a stand-alone Platt scaling baseline.},
- archiveprefix = {arXiv},
- bdsk-url-1 = {http://arxiv.org/abs/2009.14193},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- file = {:/Users/FA31DU/Zotero/storage/5BYIRBR2/Angelopoulos et al. - 2022 - Uncertainty Sets for Image Classifiers using Confo.pdf:;:/Users/FA31DU/Zotero/storage/2QJAKFKV/2009.html:},
- keywords = {Computer Science - Computer Vision and Pattern Recognition, Mathematics - Statistics Theory, Statistics - Machine Learning},
- month = sep,
- number = {arXiv:2009.14193},
- primaryclass = {cs, math, stat},
- publisher = {{arXiv}},
- year = {2022},
-}
-
-@Article{angelucci2009indirect,
- author = {Angelucci, Manuela and De Giorgi, Giacomo},
- title = {Indirect Effects of an Aid Program: How Do Cash Transfers Affect Ineligibles' Consumption?},
- number = {1},
- pages = {486--508},
- volume = {99},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {American economic review},
- year = {2009},
-}
-
-@Article{angrist1990lifetime,
- author = {Angrist, Joshua D},
- title = {Lifetime Earnings and the {{Vietnam}} Era Draft Lottery: Evidence from Social Security Administrative Records},
- pages = {313--336},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The American Economic Review},
- year = {1990},
-}
-
-@Unpublished{antoran2020getting,
- author = {Antor{\'a}n, Javier and Bhatt, Umang and Adel, Tameem and Weller, Adrian and Hern{\'a}ndez-Lobato, Jos{\'e} Miguel},
- title = {Getting a Clue: {{A}} Method for Explaining Uncertainty Estimates},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2006.06848},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Article{arcones1992bootstrap,
- author = {Arcones, Miguel A and Gine, Evarist},
- title = {On the Bootstrap of {{U}} and {{V}} Statistics},
- pages = {655--674},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Annals of Statistics},
- year = {1992},
-}
-
-@Article{ariely2003coherent,
- author = {Ariely, Dan and Loewenstein, George and Prelec, Drazen},
- title = {``{{Coherent}} Arbitrariness'': {{Stable}} Demand Curves without Stable Preferences},
- number = {1},
- pages = {73--106},
- volume = {118},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Quarterly journal of economics},
- year = {2003},
-}
-
-@Article{ariely2006tom,
- author = {Ariely, Dan and Loewenstein, George and Prelec, Drazen},
- title = {Tom {{Sawyer}} and the Construction of Value},
- number = {1},
- pages = {1--10},
- volume = {60},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Economic Behavior \& Organization},
- year = {2006},
-}
-
-@Article{arrieta2020explainable,
- author = {Arrieta, Alejandro Barredo and Diaz-Rodriguez, Natalia and Del Ser, Javier and Bennetot, Adrien and Tabik, Siham and Barbado, Alberto and Garcia, Salvador and Gil-Lopez, Sergio and Molina, Daniel and Benjamins, Richard and others},
- title = {Explainable {{Artificial Intelligence}} ({{XAI}}): {{Concepts}}, Taxonomies, Opportunities and Challenges toward Responsible {{AI}}},
- pages = {82--115},
- volume = {58},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Information Fusion},
- year = {2020},
-}
-
-@Article{auer2002finitetime,
- author = {Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
- title = {Finite-Time Analysis of the Multiarmed Bandit Problem},
- number = {2},
- pages = {235--256},
- volume = {47},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Machine learning},
- year = {2002},
-}
-
-@Article{barabasi2016network,
- author = {Barab{\'a}si, Albert-L{\'a}szl{\'o}},
- title = {Network {{Science}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Network Science},
- year = {2016},
-}
-
-@Unpublished{bastounis2021mathematics,
- author = {Bastounis, Alexander and Hansen, Anders C and Vla{\v c}i{\'c}, Verner},
- title = {The Mathematics of Adversarial Attacks in {{AI}}--{{Why}} Deep Learning Is Unstable despite the Existence of Stable Neural Networks},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2109.06098},
- eprinttype = {arxiv},
- year = {2021},
-}
-
-@Article{bechara1997deciding,
- author = {Bechara, Antoine and Damasio, Hanna and Tranel, Daniel and Damasio, Antonio R},
- title = {Deciding Advantageously before Knowing the Advantageous Strategy},
- number = {5304},
- pages = {1293--1295},
- volume = {275},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Science (New York, N.Y.)},
- shortjournal = {Science},
- year = {1997},
-}
-
-@Book{berlinet2011reproducing,
- author = {Berlinet, Alain and Thomas-Agnan, Christine},
- title = {Reproducing Kernel {{Hilbert}} Spaces in Probability and Statistics},
- publisher = {{Springer Science \& Business Media}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2011},
-}
-
-@Misc{bernanke1990federal,
- author = {Bernanke, Ben S},
- title = {The Federal Funds Rate and the Channels of Monetary Transnission},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- publisher = {{National Bureau of Economic Research Cambridge, Mass., USA}},
- year = {1990},
-}
-
-@Article{besbes2014stochastic,
- author = {Besbes, Omar and Gur, Yonatan and Zeevi, Assaf},
- title = {Stochastic Multi-Armed-Bandit Problem with Non-Stationary Rewards},
- pages = {199--207},
- volume = {27},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in neural information processing systems},
- year = {2014},
-}
-
-@Article{bholat2020impact,
- author = {Bholat, D and Gharbawi, M and Thew, O},
- title = {The {{Impact}} of {{Covid}} on {{Machine Learning}} and {{Data Science}} in {{UK Banking}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Bank of England Quarterly Bulletin, Q4},
- year = {2020},
-}
-
-@Book{bishop2006pattern,
- author = {Bishop, Christopher M},
- title = {Pattern Recognition and Machine Learning},
- publisher = {{springer}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2006},
-}
-
-@Article{blaom2020mlj,
- author = {Blaom, Anthony D. and Kiraly, Franz and Lienart, Thibaut and Simillides, Yiannis and Arenas, Diego and Vollmer, Sebastian J.},
- title = {{{MLJ}}: {{A Julia}} Package for Composable Machine Learning},
- doi = {10.21105/joss.02704},
- issn = {2475-9066},
- number = {55},
- pages = {2704},
- urldate = {2022-10-27},
- volume = {5},
- abstract = {Blaom et al., (2020). MLJ: A Julia package for composable machine learning. Journal of Open Source Software, 5(55), 2704, https://doi.org/10.21105/joss.02704},
- bdsk-url-1 = {https://joss.theoj.org/papers/10.21105/joss.02704},
- bdsk-url-2 = {https://doi.org/10.21105/joss.02704},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- file = {:/Users/FA31DU/Zotero/storage/7AY87FGP/Blaom et al. - 2020 - MLJ A Julia package for composable machine learni.pdf:;:/Users/FA31DU/Zotero/storage/D69YSMVF/joss.html:},
- journal = {Journal of Open Source Software},
- langid = {english},
- month = nov,
- shorttitle = {{{MLJ}}},
- year = {2020},
-}
-
-@InProceedings{blundell2015weight,
- author = {Blundell, Charles and Cornebise, Julien and Kavukcuoglu, Koray and Wierstra, Daan},
- booktitle = {International Conference on Machine Learning},
- title = {Weight Uncertainty in Neural Network},
- pages = {1613--1622},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2015},
-}
-
-@Article{borch2022machine,
- author = {Borch, Christian},
- title = {Machine Learning, Knowledge Risk, and Principal-Agent Problems in Automated Trading},
- pages = {101852},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Technology in Society},
- year = {2022},
-}
-
-@Unpublished{borisov2021deep,
- author = {Borisov, Vadim and Leemann, Tobias and Se{\ss}ler, Kathrin and Haug, Johannes and Pawelczyk, Martin and Kasneci, Gjergji},
- title = {Deep Neural Networks and Tabular Data: {{A}} Survey},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2110.01889},
- eprinttype = {arxiv},
- year = {2021},
-}
-
-@Article{bramoulle2009identification,
- author = {Bramoull{\'e}, Yann and Djebbari, Habiba and Fortin, Bernard},
- title = {Identification of Peer Effects through Social Networks},
- number = {1},
- pages = {41--55},
- volume = {150},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of econometrics},
- year = {2009},
-}
-
-@Article{bramoulle2020peer,
- author = {Bramoull{\'e}, Yann and Djebbari, Habiba and Fortin, Bernard},
- title = {Peer Effects in Networks: {{A}} Survey},
- pages = {603--629},
- volume = {12},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Annual Review of Economics},
- year = {2020},
-}
-
-@Unpublished{branco2015survey,
- author = {Branco, Paula and Torgo, Luis and Ribeiro, Rita},
- title = {A Survey of Predictive Modelling under Imbalanced Distributions},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1505.01658},
- eprinttype = {arxiv},
- year = {2015},
-}
-
-@Book{brock1991nonlinear,
- author = {Brock, William Allen and Brock, William A and Hsieh, David Arthur and LeBaron, Blake Dean and Brock, William E},
- title = {Nonlinear Dynamics, Chaos, and Instability: Statistical Theory and Economic Evidence},
- publisher = {{MIT press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1991},
-}
-
-@InProceedings{buolamwini2018gender,
- author = {Buolamwini, Joy and Gebru, Timnit},
- booktitle = {Conference on Fairness, Accountability and Transparency},
- title = {Gender Shades: {{Intersectional}} Accuracy Disparities in Commercial Gender Classification},
- pages = {77--91},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2018},
-}
-
-@Unpublished{bussmann2020neural,
- author = {Bussmann, Bart and Nys, Jannes and Latr{\'e}, Steven},
- title = {Neural {{Additive Vector Autoregression Models}} for {{Causal Discovery}} in {{Time Series Data}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2010.09429},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Report{card1993minimum,
- author = {Card, David and Krueger, Alan B},
- title = {Minimum Wages and Employment: {{A}} Case Study of the Fast Food Industry in {{New Jersey}} and {{Pennsylvania}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- school = {{National Bureau of Economic Research}},
- year = {1993},
-}
-
-@InProceedings{carlini2017evaluating,
- author = {Carlini, Nicholas and Wagner, David},
- booktitle = {2017 Ieee Symposium on Security and Privacy (Sp)},
- title = {Towards Evaluating the Robustness of Neural Networks},
- pages = {39--57},
- publisher = {{IEEE}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2017},
-}
-
-@Article{carlisle2019racist,
- author = {Carlisle, M.},
- title = {Racist Data Destruction? - a {{Boston}} Housing Dataset Controversy},
- url = {https://medium.com/@docintangible/racist-data-destruction-113e3eff54a8},
- bdsk-url-1 = {https://medium.com/@docintangible/racist-data-destruction-113e3eff54a8},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2019},
-}
-
-@Article{carrell2009does,
- author = {Carrell, Scott E and Fullerton, Richard L and West, James E},
- title = {Does Your Cohort Matter? {{Measuring}} Peer Effects in College Achievement},
- number = {3},
- pages = {439--464},
- volume = {27},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Labor Economics},
- year = {2009},
-}
-
-@Article{carrell2013natural,
- author = {Carrell, Scott E and Sacerdote, Bruce I and West, James E},
- title = {From Natural Variation to Optimal Policy? {{The}} Importance of Endogenous Peer Group Formation},
- number = {3},
- pages = {855--882},
- volume = {81},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica : journal of the Econometric Society},
- shortjournal = {Econometrica},
- year = {2013},
-}
-
-@Article{carrizosa2021generating,
- author = {Carrizosa, Emilio and Ramırez-Ayerbe, Jasone and Romero, Dolores},
- title = {Generating {{Collective Counterfactual Explanations}} in {{Score-Based Classification}} via {{Mathematical Optimization}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Article{cascarino2022explainable,
- author = {Cascarino, Giuseppe and Moscatelli, Mirko and Parlapiano, Fabio},
- title = {Explainable {{Artificial Intelligence}}: Interpreting Default Forecasting Models Based on {{Machine Learning}}},
- number = {674},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Bank of Italy Occasional Paper},
- year = {2022},
-}
-
-@Article{chandola2009anomaly,
- author = {Chandola, Varun and Banerjee, Arindam and Kumar, Vipin},
- title = {Anomaly Detection: {{A}} Survey},
- number = {3},
- pages = {1--58},
- volume = {41},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {ACM computing surveys (CSUR)},
- year = {2009},
-}
-
-@Article{chapelle2011empirical,
- author = {Chapelle, Olivier and Li, Lihong},
- title = {An Empirical Evaluation of Thompson Sampling},
- pages = {2249--2257},
- volume = {24},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in neural information processing systems},
- year = {2011},
-}
-
-@Article{chetty2011adjustment,
- author = {Chetty, Raj and Friedman, John N and Olsen, Tore and Pistaferri, Luigi},
- title = {Adjustment Costs, Firm Responses, and Micro vs. Macro Labor Supply Elasticities: {{Evidence}} from {{Danish}} Tax Records},
- number = {2},
- pages = {749--804},
- volume = {126},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The quarterly journal of economics},
- year = {2011},
-}
-
-@Article{cortes1995supportvector,
- author = {Cortes, Corinna and Vapnik, Vladimir},
- title = {Support-Vector Networks},
- number = {3},
- pages = {273--297},
- volume = {20},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Machine learning},
- year = {1995},
-}
-
-@Article{crawford2019variable,
- author = {Crawford, Lorin and Flaxman, Seth R and Runcie, Daniel E and West, Mike},
- title = {Variable Prioritization in Nonlinear Black Box Methods: {{A}} Genetic Association Case Study},
- number = {2},
- pages = {958},
- volume = {13},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The annals of applied statistics},
- year = {2019},
-}
-
-@InProceedings{dai2022counterfactual,
- author = {Dai, Xinyue and Keane, Mark T and Shalloo, Laurence and Ruelle, Elodie and Byrne, Ruth MJ},
- title = {Counterfactual Explanations for Prediction and Diagnosis in Xai},
- eventtitle = {Proceedings of the 2022 {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
- pages = {215--226},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2022},
-}
-
-@Article{danielsson2021artificial,
- author = {Danielsson, Jon and Macrae, Robert and Uthemann, Andreas},
- title = {Artificial Intelligence and Systemic Risk},
- pages = {106290},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Banking \& Finance},
- year = {2021},
-}
-
-@Article{daxberger2021laplace,
- author = {Daxberger, Erik and Kristiadi, Agustinus and Immer, Alexander and Eschenhagen, Runa and Bauer, Matthias and Hennig, Philipp},
- title = {Laplace {{Redux-Effortless Bayesian Deep Learning}}},
- volume = {34},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in Neural Information Processing Systems},
- year = {2021},
-}
-
-@Article{dehejia1999causal,
- author = {Dehejia, Rajeev H and Wahba, Sadek},
- title = {Causal Effects in Nonexperimental Studies: {{Reevaluating}} the Evaluation of Training Programs},
- number = {448},
- pages = {1053--1062},
- volume = {94},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of the American statistical Association},
- year = {1999},
-}
-
-@Article{dell2010persistent,
- author = {Dell, Melissa},
- title = {The Persistent Effects of {{Peru}}'s Mining Mita},
- number = {6},
- pages = {1863--1903},
- volume = {78},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica : journal of the Econometric Society},
- shortjournal = {Econometrica},
- year = {2010},
-}
-
-@Article{denhengst2020reinforcement,
- author = {den Hengst, Floris and Grua, Eoin Martino and el Hassouni, Ali and Hoogendoorn, Mark},
- title = {Reinforcement Learning for Personalization: {{A}} Systematic Literature Review},
- issue = {Preprint},
- pages = {1--41},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Data Science},
- options = {useprefix=true},
- year = {2020},
-}
-
-@Article{deoliveira2021framework,
- author = {de Oliveira, Raphael Mazzine Barbosa and Martens, David},
- title = {A Framework and Benchmarking Study for Counterfactual Generating Methods on Tabular Data},
- number = {16},
- pages = {7274},
- volume = {11},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Applied Sciences},
- options = {useprefix=true},
- year = {2021},
-}
-
-@Article{dhurandhar2018explanations,
- author = {Dhurandhar, Amit and Chen, Pin-Yu and Luss, Ronny and Tu, Chun-Chen and Ting, Paishun and Shanmugam, Karthikeyan and Das, Payel},
- title = {Explanations Based on the Missing: {{Towards}} Contrastive Explanations with Pertinent Negatives},
- volume = {31},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in neural information processing systems},
- year = {2018},
-}
-
-@InProceedings{dombrowski2021diffeomorphic,
- author = {Dombrowski, Ann-Kathrin and Gerken, Jan E and Kessel, Pan},
- booktitle = {{{ICML Workshop}} on {{Invertible Neural Networks}}, {{Normalizing Flows}}, and {{Explicit Likelihood Models}}},
- title = {Diffeomorphic Explanations with Normalizing Flows},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@InProceedings{dorffner1996neural,
- author = {Dorffner, Georg},
- booktitle = {Neural Network World},
- title = {Neural Networks for Time Series Processing},
- publisher = {{Citeseer}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1996},
-}
-
-@Article{epstein1979stability,
- author = {Epstein, Seymour},
- title = {The Stability of Behavior: {{I}}. {{On}} Predicting Most of the People Much of the Time.},
- number = {7},
- pages = {1097},
- volume = {37},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of personality and social psychology},
- year = {1979},
-}
-
-@Online{barocas2022fairness,
- author = {Solon Barocas and Moritz Hardt and Arvind Narayanan},
- title = {Fairness and Machine Learning},
- url = {https://fairmlbook.org/index.html},
- urldate = {2022-11-08},
- bdsk-url-1 = {https://fairmlbook.org/index.html},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- month = dec,
- year = {2022},
-}
-
-@Article{falk2006clean,
- author = {Falk, Armin and Ichino, Andrea},
- title = {Clean Evidence on Peer Effects},
- number = {1},
- pages = {39--57},
- volume = {24},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of labor economics},
- year = {2006},
-}
-
-@Unpublished{fan2020interpretability,
- author = {Fan, Fenglei and Xiong, Jinjun and Wang, Ge},
- title = {On Interpretability of Artificial Neural Networks},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2001.02522},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Article{fang2011dynamic,
- author = {Fang, Hanming and Gavazza, Alessandro},
- title = {Dynamic Inefficiencies in an Employment-Based Health Insurance System: {{Theory}} and Evidence},
- number = {7},
- pages = {3047--77},
- volume = {101},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {American Economic Review},
- year = {2011},
-}
-
-@Article{fehr2000cooperation,
- author = {Fehr, Ernst and Gachter, Simon},
- title = {Cooperation and Punishment in Public Goods Experiments},
- number = {4},
- pages = {980--994},
- volume = {90},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {American Economic Review},
- year = {2000},
-}
-
-@Article{fix1951important,
- author = {Fix, E and Hodges, J},
- title = {An Important Contribution to Nonparametric Discriminant Analysis and Density Estimation},
- number = {57},
- pages = {233--238},
- volume = {3},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {International Statistical Review},
- year = {1951},
-}
-
-@Book{friedman2008monetary,
- author = {Friedman, Milton and Schwartz, Anna Jacobson},
- title = {A Monetary History of the {{United States}}, 1867-1960},
- publisher = {{Princeton University Press}},
- volume = {14},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2008},
-}
-
-@InProceedings{gal2016dropout,
- author = {Gal, Yarin and Ghahramani, Zoubin},
- booktitle = {International Conference on Machine Learning},
- title = {Dropout as a Bayesian Approximation: {{Representing}} Model Uncertainty in Deep Learning},
- pages = {1050--1059},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2016},
-}
-
-@InProceedings{gal2017deep,
- author = {Gal, Yarin and Islam, Riashat and Ghahramani, Zoubin},
- booktitle = {International {{Conference}} on {{Machine Learning}}},
- title = {Deep Bayesian Active Learning with Image Data},
- pages = {1183--1192},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2017},
-}
-
-@Article{galizzi2019external,
- author = {Galizzi, Matteo M and Navarro-Martinez, Daniel},
- title = {On the External Validity of Social Preference Games: A Systematic Lab-Field Study},
- number = {3},
- pages = {976--1002},
- volume = {65},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Management Science},
- year = {2019},
-}
-
-@Article{gama2014survey,
- author = {Gama, Jo{\~a}o and {\v Z}liobait{\.e}, Indr{\.e} and Bifet, Albert and Pechenizkiy, Mykola and Bouchachia, Abdelhamid},
- title = {A Survey on Concept Drift Adaptation},
- number = {4},
- pages = {1--37},
- volume = {46},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {ACM computing surveys (CSUR)},
- year = {2014},
-}
-
-@Unpublished{garivier2008upperconfidence,
- author = {Garivier, Aur{\'e}lien and Moulines, Eric},
- title = {On Upper-Confidence Bound Policies for Non-Stationary Bandit Problems},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {0805.3415},
- eprinttype = {arxiv},
- year = {2008},
-}
-
-@Book{gelman2013bayesian,
- author = {Gelman, Andrew and Carlin, John B and Stern, Hal S and Dunson, David B and Vehtari, Aki and Rubin, Donald B},
- title = {Bayesian Data Analysis},
- publisher = {{CRC press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2013},
-}
-
-@Article{gilbert1998immune,
- author = {Gilbert, Daniel T and Pinel, Elizabeth C and Wilson, Timothy D and Blumberg, Stephen J and Wheatley, Thalia P},
- title = {Immune Neglect: A Source of Durability Bias in Affective Forecasting.},
- number = {3},
- pages = {617},
- volume = {75},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of personality and social psychology},
- year = {1998},
-}
-
-@Article{gneezy2006uncertainty,
- author = {Gneezy, Uri and List, John A and Wu, George},
- title = {The Uncertainty Effect: {{When}} a Risky Prospect Is Valued Less than Its Worst Possible Outcome},
- number = {4},
- pages = {1283--1309},
- volume = {121},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Quarterly Journal of Economics},
- year = {2006},
-}
-
-@InCollection{goan2020bayesian,
- author = {Goan, Ethan and Fookes, Clinton},
- booktitle = {Case {{Studies}} in {{Applied Bayesian Data Science}}},
- title = {Bayesian {{Neural Networks}}: {{An Introduction}} and {{Survey}}},
- pages = {45--87},
- publisher = {{Springer}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Article{goldsmith-pinkham2013social,
- author = {Goldsmith-Pinkham, Paul and Imbens, Guido W},
- title = {Social Networks and the Identification of Peer Effects},
- number = {3},
- pages = {253--264},
- volume = {31},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Business \& Economic Statistics},
- year = {2013},
-}
-
-@Unpublished{goodfellow2014explaining,
- author = {Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
- title = {Explaining and Harnessing Adversarial Examples},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1412.6572},
- eprinttype = {arxiv},
- year = {2014},
-}
-
-@Book{goodfellow2016deep,
- author = {Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron},
- title = {Deep {{Learning}}},
- publisher = {{MIT Press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2016},
-}
-
-@Article{goodfriend2005incredible,
- author = {Goodfriend, Marvin and King, Robert G},
- title = {The Incredible {{Volcker}} Disinflation},
- number = {5},
- pages = {981--1015},
- volume = {52},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Monetary Economics},
- year = {2005},
-}
-
-@Article{graham2017econometric,
- author = {Graham, Bryan S},
- title = {An Econometric Model of Network Formation with Degree Heterogeneity},
- number = {4},
- pages = {1033--1063},
- volume = {85},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica : journal of the Econometric Society},
- shortjournal = {Econometrica},
- year = {2017},
-}
-
-@Article{greene2012econometric,
- author = {Greene, William H},
- title = {Econometric Analysis, 71e},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Stern School of Business, New York University},
- year = {2012},
-}
-
-@Article{grether1979economic,
- author = {Grether, David M and Plott, Charles R},
- title = {Economic Theory of Choice and the Preference Reversal Phenomenon},
- number = {4},
- pages = {623--638},
- volume = {69},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The American Economic Review},
- year = {1979},
-}
-
-@Article{gretton2012kernel,
- author = {Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
- title = {A Kernel Two-Sample Test},
- number = {1},
- pages = {723--773},
- volume = {13},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Journal of Machine Learning Research},
- year = {2012},
-}
-
-@Unpublished{griffith2020name,
- author = {Griffith, Alan},
- title = {Name {{Your Friends}}, but {{Only Five}}? {{The Importance}} of {{Censoring}} in {{Peer Effects Estimates}} Using {{Social Network Data}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Unpublished{grinsztajn2022why,
- author = {Grinsztajn, L{\'e}o and Oyallon, Edouard and Varoquaux, Ga{\"e}l},
- title = {Why Do Tree-Based Models Still Outperform Deep Learning on Tabular Data?},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2207.08815},
- eprinttype = {arxiv},
- year = {2022},
-}
-
-@Misc{group2020detailed,
- author = {Group, Open COVID-19 Data Working},
- title = {Detailed {{Epidemiological Data}} from the {{COVID-19 Outbreak}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@InProceedings{gupta2011thompson,
- author = {Gupta, Neha and Granmo, Ole-Christoffer and Agrawala, Ashok},
- booktitle = {2011 10th {{International Conference}} on {{Machine Learning}} and {{Applications}} and {{Workshops}}},
- title = {Thompson Sampling for Dynamic Multi-Armed Bandits},
- pages = {484--489},
- publisher = {{IEEE}},
- volume = {1},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2011},
-}
-
-@Book{hamilton2020time,
- author = {Hamilton, James Douglas},
- title = {Time Series Analysis},
- publisher = {{Princeton university press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Article{hamon2020robustness,
- author = {Hamon, Ronan and Junklewitz, Henrik and Sanchez, Ignacio},
- title = {Robustness and Explainability of Artificial Intelligence},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Publications Office of the European Union},
- year = {2020},
-}
-
-@Article{hamzacebi2008improving,
- author = {Hamza{\c c}ebi, Co{\c s}kun},
- title = {Improving Artificial Neural Networks' Performance in Seasonal Time Series Forecasting},
- number = {23},
- pages = {4550--4559},
- volume = {178},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Information Sciences},
- year = {2008},
-}
-
-@InProceedings{hanneke2007bound,
- author = {Hanneke, Steve},
- booktitle = {Proceedings of the 24th International Conference on {{Machine}} Learning},
- title = {A Bound on the Label Complexity of Agnostic Active Learning},
- pages = {353--360},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2007},
-}
-
-@Article{hansen2020virtue,
- author = {Hansen, Kristian Bondo},
- title = {The Virtue of Simplicity: {{On}} Machine Learning Models in Algorithmic Trading},
- number = {1},
- pages = {2053951720926558},
- volume = {7},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Big Data \& Society},
- year = {2020},
-}
-
-@Article{hartland2006multiarmed,
- author = {Hartland, C{\'e}dric and Gelly, Sylvain and Baskiotis, Nicolas and Teytaud, Olivier and Sebag, Michele},
- title = {Multi-Armed Bandit, Dynamic Environments and Meta-Bandits},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2006},
-}
-
-@Article{heckman1985alternative,
- author = {Heckman, James J and Robb Jr, Richard},
- title = {Alternative Methods for Evaluating the Impact of Interventions: {{An}} Overview},
- number = {1-2},
- pages = {239--267},
- volume = {30},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of econometrics},
- year = {1985},
-}
-
-@Article{hershfield2011increasing,
- author = {Hershfield, Hal E and Goldstein, Daniel G and Sharpe, William F and Fox, Jesse and Yeykelis, Leo and Carstensen, Laura L and Bailenson, Jeremy N},
- title = {Increasing Saving Behavior through Age-Progressed Renderings of the Future Self},
- issue = {SPL},
- pages = {S23--S37},
- volume = {48},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Marketing Research},
- year = {2011},
-}
-
-@InProceedings{ho1995random,
- author = {Ho, Tin Kam},
- booktitle = {Proceedings of 3rd International Conference on Document Analysis and Recognition},
- title = {Random Decision Forests},
- pages = {278--282},
- publisher = {{IEEE}},
- volume = {1},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1995},
-}
-
-@Article{hochreiter1997long,
- author = {Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
- title = {Long Short-Term Memory},
- number = {8},
- pages = {1735--1780},
- volume = {9},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Neural computation},
- year = {1997},
-}
-
-@Unpublished{hoff2021bayesoptimal,
- author = {Hoff, Peter},
- title = {Bayes-Optimal Prediction with Frequentist Coverage Control},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2105.14045},
- eprinttype = {arxiv},
- file = {:/Users/FA31DU/Zotero/storage/IQK27WVA/Hoff - 2021 - Bayes-optimal prediction with frequentist coverage.pdf:;:/Users/FA31DU/Zotero/storage/K8EAZA25/2105.html:},
- year = {2021},
-}
-
-@Misc{hoffman1994german,
- author = {Hoffman, Hans},
- title = {German {{Credit Data}}},
- url = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
- bdsk-url-1 = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1994},
-}
-
-@Online{hoffmanGermanCreditData1994,
- author = {Hoffman, Hans},
- title = {German {{Credit Data}}},
- url = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
- bdsk-url-1 = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1994},
-}
-
-@Unpublished{houlsby2011bayesian,
- author = {Houlsby, Neil and Husz{\'a}r, Ferenc and Ghahramani, Zoubin and Lengyel, M{\'a}t{\'e}},
- title = {Bayesian Active Learning for Classification and Preference Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1112.5745},
- eprinttype = {arxiv},
- year = {2011},
-}
-
-@Article{hsee1996evaluability,
- author = {Hsee, Christopher K},
- title = {The Evaluability Hypothesis: {{An}} Explanation for Preference Reversals between Joint and Separate Evaluations of Alternatives},
- number = {3},
- pages = {247--257},
- volume = {67},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Organizational behavior and human decision processes},
- year = {1996},
-}
-
-@Article{hsee2004music,
- author = {Hsee, Christopher K and Rottenstreich, Yuval},
- title = {Music, Pandas, and Muggers: On the Affective Psychology of Value.},
- number = {1},
- pages = {23},
- volume = {133},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Experimental Psychology: General},
- year = {2004},
-}
-
-@Article{hsieh2016social,
- author = {Hsieh, Chih-Sheng and Lee, Lung Fei},
- title = {A Social Interactions Model with Endogenous Friendship Formation and Selectivity},
- number = {2},
- pages = {301--319},
- volume = {31},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Applied Econometrics},
- year = {2016},
-}
-
-@Unpublished{immer2020improving,
- author = {Immer, Alexander and Korzepa, Maciej and Bauer, Matthias},
- title = {Improving Predictions of Bayesian Neural Networks via Local Linearization},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2008.08400},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Unpublished{innes2018fashionable,
- author = {Innes, Michael and Saba, Elliot and Fischer, Keno and Gandhi, Dhairya and Rudilosso, Marco Concetto and Joy, Neethu Mariya and Karmali, Tejan and Pal, Avik and Shah, Viral},
- title = {Fashionable Modelling with Flux},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1811.01457},
- eprinttype = {arxiv},
- year = {2018},
-}
-
-@Article{innes2018flux,
- author = {Innes, Mike},
- title = {Flux: {{Elegant}} Machine Learning with {{Julia}}},
- number = {25},
- pages = {602},
- volume = {3},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Open Source Software},
- year = {2018},
-}
-
-@Unpublished{ish-horowicz2019interpreting,
- author = {Ish-Horowicz, Jonathan and Udwin, Dana and Flaxman, Seth and Filippi, Sarah and Crawford, Lorin},
- title = {Interpreting Deep Neural Networks through Variable Importance},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1901.09839},
- eprinttype = {arxiv},
- year = {2019},
-}
-
-@InProceedings{jabbari2017fairness,
- author = {Jabbari, Shahin and Joseph, Matthew and Kearns, Michael and Morgenstern, Jamie and Roth, Aaron},
- booktitle = {International {{Conference}} on {{Machine Learning}}},
- title = {Fairness in Reinforcement Learning},
- pages = {1617--1626},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2017},
-}
-
-@Article{jackson2007meeting,
- author = {Jackson, Matthew O and Rogers, Brian W},
- title = {Meeting Strangers and Friends of Friends: {{How}} Random Are Social Networks?},
- number = {3},
- pages = {890--915},
- volume = {97},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {American Economic Review},
- year = {2007},
-}
-
-@Unpublished{jeanneret2022diffusion,
- author = {Jeanneret, Guillaume and Simon, Lo{\"\i}c and Jurie, Fr{\'e}d{\'e}ric},
- title = {Diffusion {{Models}} for {{Counterfactual Explanations}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2203.15636},
- eprinttype = {arxiv},
- year = {2022},
-}
-
-@Article{johansson2005failure,
- author = {Johansson, Petter and Hall, Lars and Sikstr{\"o}m, Sverker and Olsson, Andreas},
- title = {Failure to Detect Mismatches between Intention and Outcome in a Simple Decision Task},
- number = {5745},
- pages = {116--119},
- volume = {310},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Science (New York, N.Y.)},
- shortjournal = {Science},
- year = {2005},
-}
-
-@Article{johnsson2021estimation,
- author = {Johnsson, Ida and Moon, Hyungsik Roger},
- title = {Estimation of Peer Effects in Endogenous Social Networks: {{Control}} Function Approach},
- number = {2},
- pages = {328--345},
- volume = {103},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Review of Economics and Statistics},
- year = {2021},
-}
-
-@Article{jolliffe2003modified,
- author = {Jolliffe, Ian T and Trendafilov, Nickolay T and Uddin, Mudassir},
- title = {A Modified Principal Component Technique Based on the {{LASSO}}},
- number = {3},
- pages = {531--547},
- volume = {12},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of computational and Graphical Statistics},
- year = {2003},
-}
-
-@Article{joseph2021forecasting,
- author = {Joseph, Andreas and Kalamara, Eleni and Kapetanios, George and Potjagailo, Galina},
- title = {Forecasting Uk Inflation Bottom Up},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Unpublished{joshi2019realistic,
- author = {Joshi, Shalmali and Koyejo, Oluwasanmi and Vijitbenjaronk, Warut and Kim, Been and Ghosh, Joydeep},
- title = {Towards Realistic Individual Recourse and Actionable Explanations in Black-Box Decision Making Systems},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1907.09615},
- eprinttype = {arxiv},
- year = {2019},
-}
-
-@Unpublished{jospin2020handson,
- author = {Jospin, Laurent Valentin and Buntine, Wray and Boussaid, Farid and Laga, Hamid and Bennamoun, Mohammed},
- title = {Hands-on {{Bayesian Neural Networks}}--a {{Tutorial}} for {{Deep Learning Users}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2007.06823},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Misc{kaggle2011give,
- author = {Kaggle},
- title = {Give Me Some Credit, {{Improve}} on the State of the Art in Credit Scoring by Predicting the Probability That Somebody Will Experience Financial Distress in the next Two Years.},
- url = {https://www.kaggle.com/c/GiveMeSomeCredit},
- bdsk-url-1 = {https://www.kaggle.com/c/GiveMeSomeCredit},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- publisher = {{Kaggle}},
- year = {2011},
-}
-
-@online{kagglecompetitionGiveMeCredit,
- author = {Kaggle Competition},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- title = {Give Me Some Credit, {{Improve}} on the State of the Art in Credit Scoring by Predicting the Probability That Somebody Will Experience Financial Distress in the next Two Years.},
- url = {https://www.kaggle.com/c/GiveMeSomeCredit},
- bdsk-url-1 = {https://www.kaggle.com/c/GiveMeSomeCredit}}
-
-@Article{kahneman1979prospect,
- author = {Kahneman, Daniel and Tversky, Amos},
- title = {Prospect {{Theory}}: {{An Analysis}} of {{Decision}} under {{Risk}}},
- pages = {263--291},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica: Journal of the Econometric Society},
- year = {1979},
-}
-
-@Article{kahneman1990experimental,
- author = {Kahneman, Daniel and Knetsch, Jack L and Thaler, Richard H},
- title = {Experimental Tests of the Endowment Effect and the {{Coase}} Theorem},
- number = {6},
- pages = {1325--1348},
- volume = {98},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of political Economy},
- year = {1990},
-}
-
-@Article{kahneman1992reference,
- author = {Kahneman, Daniel},
- title = {Reference Points, Anchors, Norms, and Mixed Feelings},
- number = {2},
- pages = {296--312},
- volume = {51},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Organizational behavior and human decision processes},
- year = {1992},
-}
-
-@Unpublished{karimi2020algorithmic,
- author = {Karimi, Amir-Hossein and Von K{\"u}gelgen, Julius and Sch{\"o}lkopf, Bernhard and Valera, Isabel},
- title = {Algorithmic Recourse under Imperfect Causal Knowledge: A Probabilistic Approach},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2006.06831},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Unpublished{karimi2020survey,
- author = {Karimi, Amir-Hossein and Barthe, Gilles and Sch{\"o}lkopf, Bernhard and Valera, Isabel},
- title = {A Survey of Algorithmic Recourse: Definitions, Formulations, Solutions, and Prospects},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2010.04050},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@InProceedings{karimi2021algorithmic,
- author = {Karimi, Amir-Hossein and Sch{\"o}lkopf, Bernhard and Valera, Isabel},
- booktitle = {Proceedings of the 2021 {{ACM Conference}} on {{Fairness}}, {{Accountability}}, and {{Transparency}}},
- title = {Algorithmic Recourse: From Counterfactual Explanations to Interventions},
- pages = {353--362},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@InProceedings{kaur2020interpreting,
- author = {Kaur, Harmanpreet and Nori, Harsha and Jenkins, Samuel and Caruana, Rich and Wallach, Hanna and Wortman Vaughan, Jennifer},
- booktitle = {Proceedings of the 2020 {{CHI}} Conference on Human Factors in Computing Systems},
- title = {Interpreting Interpretability: Understanding Data Scientists' Use of Interpretability Tools for Machine Learning},
- pages = {1--14},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Article{kehoe2021defence,
- author = {Kehoe, Aidan and Wittek, Peter and Xue, Yanbo and Pozas-Kerstjens, Alejandro},
- title = {Defence against Adversarial Attacks Using Classical and Quantum-Enhanced {{Boltzmann}} Machines},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Machine Learning: Science and Technology},
- year = {2021},
-}
-
-@Unpublished{kendall2017what,
- author = {Kendall, Alex and Gal, Yarin},
- title = {What Uncertainties Do We Need in Bayesian Deep Learning for Computer Vision?},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1703.04977},
- eprinttype = {arxiv},
- year = {2017},
-}
-
-@Article{kihoro2004seasonal,
- author = {Kihoro, J and Otieno, RO and Wafula, C},
- title = {Seasonal Time Series Forecasting: {{A}} Comparative Study of {{ARIMA}} and {{ANN}} Models},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2004},
-}
-
-@Book{kilian2017structural,
- author = {Kilian, Lutz and L{\"u}tkepohl, Helmut},
- title = {Structural Vector Autoregressive Analysis},
- publisher = {{Cambridge University Press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2017},
-}
-
-@Unpublished{kingma2014adam,
- author = {Kingma, Diederik P and Ba, Jimmy},
- title = {Adam: {{A}} Method for Stochastic Optimization},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1412.6980},
- eprinttype = {arxiv},
- year = {2014},
-}
-
-@Article{kirsch2019batchbald,
- author = {Kirsch, Andreas and Van Amersfoort, Joost and Gal, Yarin},
- title = {Batchbald: {{Efficient}} and Diverse Batch Acquisition for Deep Bayesian Active Learning},
- pages = {7026--7037},
- volume = {32},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in neural information processing systems},
- year = {2019},
-}
-
-@Unpublished{kuiper2021exploring,
- author = {Kuiper, Ouren and van den Berg, Martin and van den Burgt, Joost and Leijnen, Stefan},
- title = {Exploring {{Explainable AI}} in the {{Financial Sector}}: {{Perspectives}} of {{Banks}} and {{Supervisory Authorities}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2111.02244},
- eprinttype = {arxiv},
- year = {2021},
-}
-
-@Article{kydland1982time,
- author = {Kydland, Finn E and Prescott, Edward C},
- title = {Time to Build and Aggregate Fluctuations},
- pages = {1345--1370},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica: Journal of the Econometric Society},
- year = {1982},
-}
-
-@Unpublished{lachapelle2019gradientbased,
- author = {Lachapelle, S{\'e}bastien and Brouillard, Philippe and Deleu, Tristan and Lacoste-Julien, Simon},
- title = {Gradient-Based Neural Dag Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1906.02226},
- eprinttype = {arxiv},
- year = {2019},
-}
-
-@InProceedings{lakkaraju2020how,
- author = {Lakkaraju, Himabindu and Bastani, Osbert},
- booktitle = {Proceedings of the {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
- title = {" {{How}} Do {{I}} Fool You?" {{Manipulating User Trust}} via {{Misleading Black Box Explanations}}},
- pages = {79--85},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@InProceedings{lakkaraju2020how,
- author = {Lakkaraju, Himabindu and Bastani, Osbert},
- booktitle = {Proceedings of the {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
- title = {" {{How Do I Fool You}}?" {{Manipulating User Trust}} via {{Misleading Black Box Explanations}}},
- pages = {79--85},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Unpublished{lakshminarayanan2016simple,
- author = {Lakshminarayanan, Balaji and Pritzel, Alexander and Blundell, Charles},
- title = {Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1612.01474},
- eprinttype = {arxiv},
- year = {2016},
-}
-
-@Unpublished{laugel2017inverse,
- author = {Laugel, Thibault and Lesot, Marie-Jeanne and Marsala, Christophe and Renard, Xavier and Detyniecki, Marcin},
- title = {Inverse Classification for Comparison-Based Interpretability in Machine Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1712.08443},
- eprinttype = {arxiv},
- shortjournal = {arXiv preprint arXiv:1712.08443},
- year = {2017},
-}
-
-@Thesis{lawrence2001variational,
- author = {Lawrence, Neil David},
- title = {Variational Inference in Probabilistic Models},
- type = {phdthesis},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- school = {{University of Cambridge}},
- year = {2001},
-}
-
-@Article{lecun1998mnist,
- author = {LeCun, Yann},
- title = {The {{MNIST}} Database of Handwritten Digits},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- shortjournal = {http://yann. lecun. com/exdb/mnist/},
- year = {1998},
-}
-
-@Article{lee2003best,
- author = {Lee, Lung-fei},
- title = {Best Spatial Two-Stage Least Squares Estimators for a Spatial Autoregressive Model with Autoregressive Disturbances},
- number = {4},
- pages = {307--335},
- volume = {22},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometric Reviews},
- year = {2003},
-}
-
-@Article{lerner2013financial,
- author = {Lerner, Jennifer S and Li, Ye and Weber, Elke U},
- title = {The Financial Costs of Sadness},
- number = {1},
- pages = {72--79},
- volume = {24},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Psychological science},
- year = {2013},
-}
-
-@Article{list2004neoclassical,
- author = {List, John A},
- title = {Neoclassical Theory versus Prospect Theory: {{Evidence}} from the Marketplace},
- number = {2},
- pages = {615--625},
- volume = {72},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Econometrica : journal of the Econometric Society},
- shortjournal = {Econometrica},
- year = {2004},
-}
-
-@Article{lucas1976econometric,
- author = {Lucas, JR},
- title = {Econometric Policy Evaluation: A Critique `, in {{K}}. {{Brunner}} and {{A Meltzer}}, {{The Phillips}} Curve and Labor Markets, {{North Holland}}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {1976},
-}
-
-@InProceedings{lundberg2017unified,
- author = {Lundberg, Scott M and Lee, Su-In},
- booktitle = {Proceedings of the 31st International Conference on Neural Information Processing Systems},
- title = {A Unified Approach to Interpreting Model Predictions},
- pages = {4768--4777},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2017},
-}
-
-@Book{lutkepohl2005new,
- author = {L{\"u}tkepohl, Helmut},
- title = {New Introduction to Multiple Time Series Analysis},
- publisher = {{Springer Science \& Business Media}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2005},
-}
-
-@Article{madrian2001power,
- author = {Madrian, Brigitte C and Shea, Dennis F},
- title = {The Power of Suggestion: {{Inertia}} in 401 (k) Participation and Savings Behavior},
- number = {4},
- pages = {1149--1187},
- volume = {116},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Quarterly journal of economics},
- year = {2001},
-}
-
-@Book{manning2008introduction,
- author = {Manning, Christopher D and Sch{\"u}tze, Hinrich and Raghavan, Prabhakar},
- title = {Introduction to Information Retrieval},
- publisher = {{Cambridge university press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2008},
-}
-
-@misc{manokhin2022awesome,
- author = {Manokhin, Valery},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- title = {Awesome Conformal Prediction}}
-
-@Article{manski1993identification,
- author = {Manski, Charles F},
- title = {Identification of Endogenous Social Effects: {{The}} Reflection Problem},
- number = {3},
- pages = {531--542},
- volume = {60},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The review of economic studies},
- year = {1993},
-}
-
-@Article{markle2018goals,
- author = {Markle, Alex and Wu, George and White, Rebecca and Sackett, Aaron},
- title = {Goals as Reference Points in Marathon Running: {{A}} Novel Test of Reference Dependence},
- number = {1},
- pages = {19--50},
- volume = {56},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Risk and Uncertainty},
- year = {2018},
-}
-
-@Article{masini2021machine,
- author = {Masini, Ricardo P and Medeiros, Marcelo C and Mendes, Eduardo F},
- title = {Machine Learning Advances for Time Series Forecasting},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Economic Surveys},
- year = {2021},
-}
-
-@Article{mccracken2016fredmd,
- author = {McCracken, Michael W and Ng, Serena},
- title = {{{FRED-MD}}: {{A}} Monthly Database for Macroeconomic Research},
- number = {4},
- pages = {574--589},
- volume = {34},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Business \& Economic Statistics},
- year = {2016},
-}
-
-@Article{mcculloch1990logical,
- author = {McCulloch, Warren S and Pitts, Walter},
- title = {A Logical Calculus of the Ideas Immanent in Nervous Activity},
- number = {1},
- pages = {99--115},
- volume = {52},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Bulletin of mathematical biology},
- year = {1990},
-}
-
-@Article{migut2015visualizing,
- author = {Migut, MA and Worring, Marcel and Veenman, Cor J},
- title = {Visualizing Multi-Dimensional Decision Boundaries in {{2D}}},
- number = {1},
- pages = {273--295},
- volume = {29},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Data Mining and Knowledge Discovery},
- year = {2015},
-}
-
-@Article{miller2019explanation,
- author = {Miller, Tim},
- title = {Explanation in Artificial Intelligence: {{Insights}} from the Social Sciences},
- pages = {1--38},
- volume = {267},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Artificial intelligence},
- year = {2019},
-}
-
-@InProceedings{miller2020strategic,
- author = {Miller, John and Milli, Smitha and Hardt, Moritz},
- booktitle = {Proceedings of the 37th {{International Conference}} on {{Machine Learning}}},
- title = {Strategic {{Classification}} Is {{Causal Modeling}} in {{Disguise}}},
- eventtitle = {International {{Conference}} on {{Machine Learning}}},
- pages = {6917--6926},
- publisher = {{PMLR}},
- url = {https://proceedings.mlr.press/v119/miller20b.html},
- urldate = {2022-11-03},
- abstract = {Consequential decision-making incentivizes individuals to strategically adapt their behavior to the specifics of the decision rule. While a long line of work has viewed strategic adaptation as gaming and attempted to mitigate its effects, recent work has instead sought to design classifiers that incentivize individuals to improve a desired quality. Key to both accounts is a cost function that dictates which adaptations are rational to undertake. In this work, we develop a causal framework for strategic adaptation. Our causal perspective clearly distinguishes between gaming and improvement and reveals an important obstacle to incentive design. We prove any procedure for designing classifiers that incentivize improvement must inevitably solve a non-trivial causal inference problem. We show a similar result holds for designing cost functions that satisfy the requirements of previous work. With the benefit of hindsight, our results show much of the prior work on strategic classification is causal modeling in disguise.},
- bdsk-url-1 = {https://proceedings.mlr.press/v119/miller20b.html},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- file = {:/Users/FA31DU/Zotero/storage/46I2QMPI/Miller et al. - 2020 - Strategic Classification is Causal Modeling in Dis.pdf:;:/Users/FA31DU/Zotero/storage/NWREET6B/Miller et al. - 2020 - Strategic Classification is Causal Modeling in Dis.pdf:},
- issn = {2640-3498},
- langid = {english},
- month = nov,
- year = {2020},
-}
-
-@Article{mischel1988nature,
- author = {Mischel, Walter and Shoda, Yuichi and Peake, Philip K},
- title = {The Nature of Adolescent Competencies Predicted by Preschool Delay of Gratification.},
- number = {4},
- pages = {687},
- volume = {54},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of personality and social psychology},
- year = {1988},
-}
-
-@InProceedings{mittelstadt2019explaining,
- author = {Mittelstadt, Brent and Russell, Chris and Wachter, Sandra},
- booktitle = {Proceedings of the Conference on Fairness, Accountability, and Transparency},
- title = {Explaining Explanations in {{AI}}},
- pages = {279--288},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2019},
-}
-
-@Book{molnar2020interpretable,
- author = {Molnar, Christoph},
- title = {Interpretable Machine Learning},
- publisher = {{Lulu. com}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Book{morgan2015counterfactuals,
- author = {Morgan, Stephen L and Winship, Christopher},
- title = {Counterfactuals and Causal Inference},
- publisher = {{Cambridge University Press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2015},
-}
-
-@Article{mosteller1951experimental,
- author = {Mosteller, Frederick and Nogee, Philip},
- title = {An Experimental Measurement of Utility},
- number = {5},
- pages = {371--404},
- volume = {59},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Political Economy},
- year = {1951},
-}
-
-@InProceedings{mothilal2020explaining,
- author = {Mothilal, Ramaravind K and Sharma, Amit and Tan, Chenhao},
- booktitle = {Proceedings of the 2020 {{Conference}} on {{Fairness}}, {{Accountability}}, and {{Transparency}}},
- title = {Explaining Machine Learning Classifiers through Diverse Counterfactual Explanations},
- pages = {607--617},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Book{murphy2012machine,
- author = {Murphy, Kevin P},
- title = {Machine Learning: A Probabilistic Perspective},
- publisher = {{MIT press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2012},
-}
-
-@Book{murphy2012machine,
- author = {Murphy, Kevin P},
- title = {Machine Learning: {{A}} Probabilistic Perspective},
- publisher = {{MIT press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2012},
-}
-
-@Book{murphy2022probabilistic,
- author = {Murphy, Kevin P},
- title = {Probabilistic {{Machine Learning}}: {{An}} Introduction},
- publisher = {{MIT Press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2022},
-}
-
-@Article{nagel1995unraveling,
- author = {Nagel, Rosemarie},
- title = {Unraveling in Guessing Games: {{An}} Experimental Study},
- number = {5},
- pages = {1313--1326},
- volume = {85},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The American Economic Review},
- year = {1995},
-}
-
-@Unpublished{navarro-martinez2021bridging,
- author = {Navarro-Martinez, Daniel and Wang, Xinghua},
- title = {Bridging the Gap between the Lab and the Field: {{Dictator}} Games and Donations},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@InProceedings{nelson2015evaluating,
- author = {Nelson, Kevin and Corbin, George and Anania, Mark and Kovacs, Matthew and Tobias, Jeremy and Blowers, Misty},
- booktitle = {2015 {{IEEE Symposium}} on {{Computational Intelligence}} for {{Security}} and {{Defense Applications}} ({{CISDA}})},
- title = {Evaluating Model Drift in Machine Learning Algorithms},
- pages = {1--8},
- publisher = {{IEEE}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2015},
-}
-
-@Book{nocedal2006numerical,
- author = {Nocedal, Jorge and Wright, Stephen},
- title = {Numerical Optimization},
- publisher = {{Springer Science \& Business Media}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2006},
-}
-
-@Misc{oecd2021artificial,
- author = {{OECD}},
- title = {Artificial {{Intelligence}}, {{Machine Learning}} and {{Big Data}} in {{Finance}}: {{Opportunities}}, {{Challenges}} and {{Implications}} for {{Policy Makers}}},
- url = {https://www.oecd.org/finance/financial-markets/Artificial-intelligence-machine-learning-big-data-in-finance.pdf},
- bdsk-url-1 = {https://www.oecd.org/finance/financial-markets/Artificial-intelligence-machine-learning-big-data-in-finance.pdf},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Online{oecdArtificialIntelligenceMachine2021,
- author = {{OECD}},
- title = {Artificial {{Intelligence}}, {{Machine Learning}} and {{Big Data}} in {{Finance}}: {{Opportunities}}, {{Challenges}} and {{Implications}} for {{Policy Makers}}},
- url = {https://www.oecd.org/finance/financial-markets/Artificial-intelligence-machine-learning-big-data-in-finance.pdf},
- bdsk-url-1 = {https://www.oecd.org/finance/financial-markets/Artificial-intelligence-machine-learning-big-data-in-finance.pdf},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- publisher = {{OECD}},
- year = {2021},
-}
-
-@Book{oneil2016weapons,
- author = {O'Neil, Cathy},
- title = {Weapons of Math Destruction: {{How}} Big Data Increases Inequality and Threatens Democracy},
- publisher = {{Crown}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2016},
-}
-
-@Article{pace1997sparse,
- author = {Pace, R Kelley and Barry, Ronald},
- title = {Sparse Spatial Autoregressions},
- number = {3},
- pages = {291--297},
- volume = {33},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Statistics \& Probability Letters},
- year = {1997},
-}
-
-@Unpublished{parr2018matrix,
- author = {Parr, Terence and Howard, Jeremy},
- title = {The Matrix Calculus You Need for Deep Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1802.01528},
- eprinttype = {arxiv},
- year = {2018},
-}
-
-@Unpublished{pawelczyk2021carla,
- author = {Pawelczyk, Martin and Bielawski, Sascha and van den Heuvel, Johannes and Richter, Tobias and Kasneci, Gjergji},
- title = {Carla: A Python Library to Benchmark Algorithmic Recourse and Counterfactual Explanation Algorithms},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2108.00783},
- eprinttype = {arxiv},
- year = {2021},
-}
-
-@Book{pearl2018book,
- author = {Pearl, Judea and Mackenzie, Dana},
- title = {The Book of Why: The New Science of Cause and Effect},
- publisher = {{Basic books}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2018},
-}
-
-@Article{pearl2019seven,
- author = {Pearl, Judea},
- title = {The Seven Tools of Causal Inference, with Reflections on Machine Learning},
- number = {3},
- pages = {54--60},
- volume = {62},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Communications of the ACM},
- year = {2019},
-}
-
-@Article{pedregosa2011scikitlearn,
- author = {Pedregosa, Fabian and Varoquaux, Ga{\"e}l and Gramfort, Alexandre and Michel, Vincent and Thirion, Bertrand and Grisel, Olivier and Blondel, Mathieu and Prettenhofer, Peter and Weiss, Ron and Dubourg, Vincent and others},
- title = {Scikit-Learn: {{Machine}} Learning in {{Python}}},
- pages = {2825--2830},
- volume = {12},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {the Journal of machine Learning research},
- year = {2011},
-}
-
-@Book{perry2010economic,
- author = {Perry, George L and Tobin, James},
- title = {Economic {{Events}}, {{Ideas}}, and {{Policies}}: The 1960s and After},
- publisher = {{Brookings Institution Press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2010},
-}
-
-@Article{pfaff2008var,
- author = {Pfaff, Bernhard and others},
- title = {{{VAR}}, {{SVAR}} and {{SVEC}} Models: {{Implementation}} within {{R}} Package Vars},
- number = {4},
- pages = {1--32},
- volume = {27},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Statistical Software},
- year = {2008},
-}
-
-@Book{pindyck2014microeconomics,
- author = {Pindyck, Robert S and Rubinfeld, Daniel L},
- title = {Microeconomics},
- publisher = {{Pearson Education}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2014},
-}
-
-@Article{pope2011numbers,
- author = {Pope, Devin and Simonsohn, Uri},
- title = {Round Numbers as Goals: {{Evidence}} from Baseball, {{SAT}} Takers, and the Lab},
- number = {1},
- pages = {71--79},
- volume = {22},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Psychological science},
- year = {2011},
-}
-
-@InProceedings{poyiadzi2020face,
- author = {Poyiadzi, Rafael and Sokol, Kacper and Santos-Rodriguez, Raul and De Bie, Tijl and Flach, Peter},
- booktitle = {Proceedings of the {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
- title = {{{FACE}}: {{Feasible}} and Actionable Counterfactual Explanations},
- pages = {344--350},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Article{qu2015estimating,
- author = {Qu, Xi and Lee, Lung-fei},
- title = {Estimating a Spatial Autoregressive Model with an Endogenous Spatial Weight Matrix},
- number = {2},
- pages = {209--232},
- volume = {184},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of Econometrics},
- year = {2015},
-}
-
-@Article{rabanser2019failing,
- author = {Rabanser, Stephan and G{\"u}nnemann, Stephan and Lipton, Zachary},
- title = {Failing Loudly: {{An}} Empirical Study of Methods for Detecting Dataset Shift},
- volume = {32},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in Neural Information Processing Systems},
- year = {2019},
-}
-
-@Unpublished{raghunathan2019adversarial,
- author = {Raghunathan, Aditi and Xie, Sang Michael and Yang, Fanny and Duchi, John C and Liang, Percy},
- title = {Adversarial Training Can Hurt Generalization},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1906.06032},
- eprinttype = {arxiv},
- year = {2019},
-}
-
-@Unpublished{raj2017taming,
- author = {Raj, Vishnu and Kalyani, Sheetal},
- title = {Taming Non-Stationary Bandits: {{A Bayesian}} Approach},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1707.09727},
- eprinttype = {arxiv},
- year = {2017},
-}
-
-@InProceedings{rasmussen2003gaussian,
- author = {Rasmussen, Carl Edward},
- booktitle = {Summer School on Machine Learning},
- title = {Gaussian Processes in Machine Learning},
- pages = {63--71},
- publisher = {{Springer}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2003},
-}
-
-@InProceedings{ribeiro2016why,
- author = {Ribeiro, Marco Tulio and Singh, Sameer and Guestrin, Carlos},
- booktitle = {Proceedings of the 22nd {{ACM SIGKDD}} International Conference on Knowledge Discovery and Data Mining},
- title = {"{{Why}} Should i Trust You?" {{Explaining}} the Predictions of Any Classifier},
- pages = {1135--1144},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2016},
-}
-
-@Article{romer1989does,
- author = {Romer, Christina D and Romer, David H},
- title = {Does Monetary Policy Matter? {{A}} New Test in the Spirit of {{Friedman}} and {{Schwartz}}},
- pages = {121--170},
- volume = {4},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {NBER macroeconomics annual},
- year = {1989},
-}
-
-@Article{rudin2019stop,
- author = {Rudin, Cynthia},
- title = {Stop Explaining Black Box Machine Learning Models for High Stakes Decisions and Use Interpretable Models Instead},
- number = {5},
- pages = {206--215},
- volume = {1},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Nature Machine Intelligence},
- year = {2019},
-}
-
-@Article{sacerdote2001peer,
- author = {Sacerdote, Bruce},
- title = {Peer Effects with Random Assignment: {{Results}} for {{Dartmouth}} Roommates},
- number = {2},
- pages = {681--704},
- volume = {116},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The Quarterly journal of economics},
- year = {2001},
-}
-
-@Article{sadinle2019least,
- author = {Sadinle, Mauricio and Lei, Jing and Wasserman, Larry},
- title = {Least Ambiguous Set-Valued Classifiers with Bounded Error Levels},
- number = {525},
- pages = {223--234},
- volume = {114},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- file = {:/Users/FA31DU/Zotero/storage/YXQ8N76A/Sadinle et al. - 2019 - Least ambiguous set-valued classifiers with bounde.pdf:;:/Users/FA31DU/Zotero/storage/ZHB56F3V/01621459.2017.html:},
- journal = {Journal of the American Statistical Association},
- publisher = {{Taylor \& Francis}},
- year = {2019},
-}
-
-@InProceedings{satopaa2011finding,
- author = {Satopaa, Ville and Albrecht, Jeannie and Irwin, David and Raghavan, Barath},
- booktitle = {2011 31st International Conference on Distributed Computing Systems Workshops},
- title = {Finding a" Kneedle" in a Haystack: {{Detecting}} Knee Points in System Behavior},
- pages = {166--171},
- publisher = {{IEEE}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2011},
-}
-
-@InProceedings{schut2021generating,
- author = {Schut, Lisa and Key, Oscar and Mc Grath, Rory and Costabello, Luca and Sacaleanu, Bogdan and Gal, Yarin and others},
- booktitle = {International {{Conference}} on {{Artificial Intelligence}} and {{Statistics}}},
- title = {Generating {{Interpretable Counterfactual Explanations By Implicit Minimisation}} of {{Epistemic}} and {{Aleatoric Uncertainties}}},
- pages = {1756--1764},
- publisher = {{PMLR}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2021},
-}
-
-@Book{schutze2008introduction,
- author = {Sch{\"u}tze, Hinrich and Manning, Christopher D and Raghavan, Prabhakar},
- title = {Introduction to Information Retrieval},
- publisher = {{Cambridge University Press Cambridge}},
- volume = {39},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2008},
-}
-
-@Article{shafir1993reasonbased,
- author = {Shafir, Eldar and Simonson, Itamar and Tversky, Amos},
- title = {Reason-Based Choice},
- number = {1-2},
- pages = {11--36},
- volume = {49},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Cognition},
- year = {1993},
-}
-
-@Article{simonson1989choice,
- author = {Simonson, Itamar},
- title = {Choice Based on Reasons: {{The}} Case of Attraction and Compromise Effects},
- number = {2},
- pages = {158--174},
- volume = {16},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of consumer research},
- year = {1989},
-}
-
-@Article{sims1986are,
- author = {Sims, Christopher A and others},
- title = {Are Forecasting Models Usable for Policy Analysis?},
- issue = {Win},
- pages = {2--16},
- volume = {10},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Quarterly Review},
- year = {1986},
-}
-
-@InProceedings{slack2020fooling,
- author = {Slack, Dylan and Hilgard, Sophie and Jia, Emily and Singh, Sameer and Lakkaraju, Himabindu},
- booktitle = {Proceedings of the {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
- title = {Fooling Lime and Shap: {{Adversarial}} Attacks on Post Hoc Explanation Methods},
- pages = {180--186},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
-}
-
-@Article{slack2021counterfactual,
- author = {Slack, Dylan and Hilgard, Anna and Lakkaraju, Himabindu and Singh, Sameer},
- title = {Counterfactual Explanations Can Be Manipulated},
- volume = {34},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in Neural Information Processing Systems},
- year = {2021},
-}
-
-@Article{slovic1974who,
- author = {Slovic, Paul and Tversky, Amos},
- title = {Who Accepts {{Savage}}'s Axiom?},
- number = {6},
- pages = {368--373},
- volume = {19},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Behavioral science},
- year = {1974},
-}
-
-@Unpublished{spooner2021counterfactual,
- author = {Spooner, Thomas and Dervovic, Danial and Long, Jason and Shepard, Jon and Chen, Jiahao and Magazzeni, Daniele},
- title = {Counterfactual {{Explanations}} for {{Arbitrary Regression Models}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2106.15212},
- eprinttype = {arxiv},
- shortjournal = {arXiv preprint arXiv:2106.15212},
- year = {2021},
-}
-
-@Article{srivastava2014dropout,
- author = {Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
- title = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
- number = {1},
- pages = {1929--1958},
- volume = {15},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The journal of machine learning research},
- year = {2014},
-}
-
-@Unpublished{stanton2022bayesian,
- author = {Stanton, Samuel and Maddox, Wesley and Wilson, Andrew Gordon},
- title = {Bayesian {{Optimization}} with {{Conformal Coverage Guarantees}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2210.12496},
- eprinttype = {arxiv},
- file = {:/Users/FA31DU/Zotero/storage/XFGZAB9J/Stanton et al. - 2022 - Bayesian Optimization with Conformal Coverage Guar.pdf:;:/Users/FA31DU/Zotero/storage/RPWYDPVW/2210.html:},
- year = {2022},
-}
-
-@Article{sturm2014simple,
- author = {Sturm, Bob L},
- title = {A Simple Method to Determine If a Music Information Retrieval System Is a ``Horse''},
- number = {6},
- pages = {1636--1644},
- volume = {16},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {IEEE Transactions on Multimedia},
- year = {2014},
-}
-
-@Article{sunstein2003libertarian,
- author = {Sunstein, Cass R and Thaler, Richard H},
- title = {Libertarian Paternalism Is Not an Oxymoron},
- pages = {1159--1202},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {The University of Chicago Law Review},
- year = {2003},
-}
-
-@Book{sutton2018reinforcement,
- author = {Sutton, Richard S and Barto, Andrew G},
- title = {Reinforcement Learning: {{An}} Introduction},
- publisher = {{MIT press}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2018},
-}
-
-@Unpublished{szegedy2013intriguing,
- author = {Szegedy, Christian and Zaremba, Wojciech and Sutskever, Ilya and Bruna, Joan and Erhan, Dumitru and Goodfellow, Ian and Fergus, Rob},
- title = {Intriguing Properties of Neural Networks},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1312.6199},
- eprinttype = {arxiv},
- year = {2013},
-}
-
-@Article{thaler1981empirical,
- author = {Thaler, Richard},
- title = {Some Empirical Evidence on Dynamic Inconsistency},
- number = {3},
- pages = {201--207},
- volume = {8},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Economics letters},
- year = {1981},
-}
-
-@Article{thaler2004more,
- author = {Thaler, Richard H and Benartzi, Shlomo},
- title = {Save More Tomorrow{\texttrademark}: {{Using}} Behavioral Economics to Increase Employee Saving},
- number = {S1},
- pages = {S164--S187},
- volume = {112},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of political Economy},
- year = {2004},
-}
-
-@Article{tversky1981framing,
- author = {Tversky, Amos and Kahneman, Daniel},
- title = {The Framing of Decisions and the Psychology of Choice},
- number = {4481},
- pages = {453--458},
- volume = {211},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Science (New York, N.Y.)},
- shortjournal = {science},
- year = {1981},
-}
-
-@Article{ungemach2011how,
- author = {Ungemach, Christoph and Stewart, Neil and Reimers, Stian},
- title = {How Incidental Values from the Environment Affect Decisions about Money, Risk, and Delay},
- number = {2},
- pages = {253--260},
- volume = {22},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Psychological Science},
- year = {2011},
-}
-
-@Unpublished{upadhyay2021robust,
- author = {Upadhyay, Sohini and Joshi, Shalmali and Lakkaraju, Himabindu},
- title = {Towards {{Robust}} and {{Reliable Algorithmic Recourse}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2102.13620},
- eprinttype = {arxiv},
- year = {2021},
-}
-
-@InProceedings{ustun2019actionable,
- author = {Ustun, Berk and Spangher, Alexander and Liu, Yang},
- booktitle = {Proceedings of the {{Conference}} on {{Fairness}}, {{Accountability}}, and {{Transparency}}},
- title = {Actionable Recourse in Linear Classification},
- pages = {10--19},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2019},
-}
-
-@Article{vanboven2000egocentric,
- author = {Van Boven, Leaf and Dunning, David and Loewenstein, George},
- title = {Egocentric Empathy Gaps between Owners and Buyers: Misperceptions of the Endowment Effect.},
- number = {1},
- pages = {66},
- volume = {79},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of personality and social psychology},
- year = {2000},
-}
-
-@Book{varshney2022trustworthy,
- author = {Varshney, Kush R.},
- title = {Trustworthy {{Machine Learning}}},
- publisher = {{Independently Published}},
- address = {{Chappaqua, NY, USA}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2022},
-}
-
-@Unpublished{verma2020counterfactual,
- author = {Verma, Sahil and Dickerson, John and Hines, Keegan},
- title = {Counterfactual Explanations for Machine Learning: {{A}} Review},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2010.10596},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Article{verstyuk2020modeling,
- author = {Verstyuk, Sergiy},
- title = {Modeling Multivariate Time Series in Economics: {{From}} Auto-Regressions to Recurrent Neural Networks},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Available at SSRN 3589337},
- year = {2020},
-}
-
-@Article{wachter2017counterfactual,
- author = {Wachter, Sandra and Mittelstadt, Brent and Russell, Chris},
- title = {Counterfactual Explanations without Opening the Black Box: {{Automated}} Decisions and the {{GDPR}}},
- pages = {841},
- volume = {31},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Harv. JL \& Tech.},
- year = {2017},
-}
-
-@Article{wang2018optimal,
- author = {Wang, HaiYing and Zhu, Rong and Ma, Ping},
- title = {Optimal Subsampling for Large Sample Logistic Regression},
- number = {522},
- pages = {829--844},
- volume = {113},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Journal of the American Statistical Association},
- year = {2018},
-}
-
-@Book{wasserman2006all,
- author = {Wasserman, Larry},
- title = {All of Nonparametric Statistics},
- publisher = {{Springer Science \& Business Media}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2006},
-}
-
-@Book{wasserman2013all,
- author = {Wasserman, Larry},
- title = {All of Statistics: A Concise Course in Statistical Inference},
- publisher = {{Springer Science \& Business Media}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2013},
-}
-
-@Article{widmer1996learning,
- author = {Widmer, Gerhard and Kubat, Miroslav},
- title = {Learning in the Presence of Concept Drift and Hidden Contexts},
- number = {1},
- pages = {69--101},
- volume = {23},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Machine learning},
- year = {1996},
-}
-
-@Unpublished{wilson2020case,
- author = {Wilson, Andrew Gordon},
- title = {The Case for {{Bayesian}} Deep Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2001.10995},
- eprinttype = {arxiv},
- year = {2020},
-}
-
-@Article{witten2009penalized,
- author = {Witten, Daniela M and Tibshirani, Robert and Hastie, Trevor},
- title = {A Penalized Matrix Decomposition, with Applications to Sparse Principal Components and Canonical Correlation Analysis},
- number = {3},
- pages = {515--534},
- volume = {10},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Biostatistics (Oxford, England)},
- shortjournal = {Biostatistics},
- year = {2009},
-}
-
-@Article{xu2020epidemiological,
- author = {Xu, Bo and Gutierrez, Bernardo and Mekaru, Sumiko and Sewalk, Kara and Goodwin, Lauren and Loskill, Alyssa and Cohn, Emily and Hswen, Yulin and Hill, Sarah C. and Cobo, Maria M and Zarebski, Alexander and Li, Sabrina and Wu, Chieh-Hsi and Hulland, Erin and Morgan, Julia and Wang, Lin and O'Brien, Katelynn and Scarpino, Samuel V. and Brownstein, John S. and Pybus, Oliver G. and Pigott, David M. and Kraemer, Moritz U. G.},
- title = {Epidemiological Data from the {{COVID-19}} Outbreak, Real-Time Case Information},
- doi = {doi.org/10.1038/s41597-020-0448-0},
- number = {106},
- volume = {7},
- bdsk-url-1 = {https://doi.org/10.1038/s41597-020-0448-0},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Scientific Data},
- year = {2020},
-}
-
-@Article{yeh2009comparisons,
- author = {Yeh, I-Cheng and Lien, Che-hui},
- title = {The Comparisons of Data Mining Techniques for the Predictive Accuracy of Probability of Default of Credit Card Clients},
- number = {2},
- pages = {2473--2480},
- volume = {36},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Expert systems with applications},
- year = {2009},
-}
-
-@Article{zhang1998forecasting,
- author = {Zhang, Guoqiang and Patuwo, B Eddy and Hu, Michael Y},
- title = {Forecasting with Artificial Neural Networks:: {{The}} State of the Art},
- number = {1},
- pages = {35--62},
- volume = {14},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {International journal of forecasting},
- year = {1998},
-}
-
-@Article{zhang2003time,
- author = {Zhang, G Peter},
- title = {Time Series Forecasting Using a Hybrid {{ARIMA}} and Neural Network Model},
- pages = {159--175},
- volume = {50},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Neurocomputing},
- year = {2003},
-}
-
-@Unpublished{zheng2018dags,
- author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P},
- title = {Dags with No Tears: {{Continuous}} Optimization for Structure Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1803.01422},
- eprinttype = {arxiv},
- year = {2018},
-}
-
-@Article{zhu2015optimal,
- author = {Zhu, Rong and Ma, Ping and Mahoney, Michael W and Yu, Bin},
- title = {Optimal Subsampling Approaches for Large Sample Linear Regression},
- pages = {arXiv--1509},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {arXiv},
- year = {2015},
-}
-
-@Article{barber2021predictive,
- author = {Barber, Rina Foygel and Candès, Emmanuel J. and Ramdas, Aaditya and Tibshirani, Ryan J.},
- title = {Predictive inference with the jackknife+},
- doi = {10.1214/20-AOS1965},
- issn = {0090-5364, 2168-8966},
- number = {1},
- pages = {486--507},
- urldate = {2022-12-13},
- volume = {49},
- abstract = {This paper introduces the jackknife+, which is a novel method for constructing predictive confidence intervals. Whereas the jackknife outputs an interval centered at the predicted response of a test point, with the width of the interval determined by the quantiles of leave-one-out residuals, the jackknife+ also uses the leave-one-out predictions at the test point to account for the variability in the fitted regression function. Assuming exchangeable training samples, we prove that this crucial modification permits rigorous coverage guarantees regardless of the distribution of the data points, for any algorithm that treats the training points symmetrically. Such guarantees are not possible for the original jackknife and we demonstrate examples where the coverage rate may actually vanish. Our theoretical and empirical analysis reveals that the jackknife and the jackknife+ intervals achieve nearly exact coverage and have similar lengths whenever the fitting algorithm obeys some form of stability. Further, we extend the jackknife+ to \$K\$-fold cross validation and similarly establish rigorous coverage properties. Our methods are related to cross-conformal prediction proposed by Vovk (Ann. Math. Artif. Intell. 74 (2015) 9–28) and we discuss connections.},
- file = {:Barber2021 - Predictive Inference with the Jackknife+.pdf:PDF},
- journal = {The Annals of Statistics},
- keywords = {62F40, 62G08, 62G09, conformal inference, cross-validation, distribution-free, jackknife, leave-one-out, stability},
- month = feb,
- publisher = {Institute of Mathematical Statistics},
- year = {2021},
-}
-
-@TechReport{chouldechova2018frontiers,
- author = {Chouldechova, Alexandra and Roth, Aaron},
- title = {The {Frontiers} of {Fairness} in {Machine} {Learning}},
- doi = {10.48550/arXiv.1810.08810},
- eprint = {1810.08810},
- note = {arXiv:1810.08810 [cs, stat] type: article},
- abstract = {The last few years have seen an explosion of academic and popular interest in algorithmic fairness. Despite this interest and the volume and velocity of work that has been produced recently, the fundamental science of fairness in machine learning is still in a nascent state. In March 2018, we convened a group of experts as part of a CCC visioning workshop to assess the state of the field, and distill the most promising research directions going forward. This report summarizes the findings of that workshop. Along the way, it surveys recent theoretical work in the field and points towards promising directions for research.},
- archiveprefix = {arxiv},
- file = {:chouldechova2018frontiers - The Frontiers of Fairness in Machine Learning.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Data Structures and Algorithms, Computer Science - Computer Science and Game Theory, Statistics - Machine Learning},
- month = oct,
- school = {arXiv},
- year = {2018},
-}
-
-@Article{pawelczyk2022probabilistically,
- author = {Pawelczyk, Martin and Datta, Teresa and van-den-Heuvel, Johannes and Kasneci, Gjergji and Lakkaraju, Himabindu},
- title = {Probabilistically {Robust} {Recourse}: {Navigating} the {Trade}-offs between {Costs} and {Robustness} in {Algorithmic} {Recourse}},
- file = {:pawelczyk2022probabilistically - Probabilistically Robust Recourse_ Navigating the Trade Offs between Costs and Robustness in Algorithmic Recourse.pdf:PDF},
- journal = {arXiv preprint arXiv:2203.06768},
- shorttitle = {Probabilistically {Robust} {Recourse}},
- year = {2022},
-}
-
-@InProceedings{stutz2022learning,
- author = {Stutz, David and Dvijotham, Krishnamurthy Dj and Cemgil, Ali Taylan and Doucet, Arnaud},
- title = {Learning {Optimal} {Conformal} {Classifiers}},
- language = {en},
- url = {https://openreview.net/forum?id=t8O-4LKFVx},
- urldate = {2023-02-13},
- abstract = {Modern deep learning based classifiers show very high accuracy on test data but this does not provide sufficient guarantees for safe deployment, especially in high-stake AI applications such as medical diagnosis. Usually, predictions are obtained without a reliable uncertainty estimate or a formal guarantee. Conformal prediction (CP) addresses these issues by using the classifier's predictions, e.g., its probability estimates, to predict confidence sets containing the true class with a user-specified probability. However, using CP as a separate processing step after training prevents the underlying model from adapting to the prediction of confidence sets. Thus, this paper explores strategies to differentiate through CP during training with the goal of training model with the conformal wrapper end-to-end. In our approach, conformal training (ConfTr), we specifically "simulate" conformalization on mini-batches during training. Compared to standard training, ConfTr reduces the average confidence set size (inefficiency) of state-of-the-art CP methods applied after training. Moreover, it allows to "shape" the confidence sets predicted at test time, which is difficult for standard CP. On experiments with several datasets, we show ConfTr can influence how inefficiency is distributed across classes, or guide the composition of confidence sets in terms of the included classes, while retaining the guarantees offered by CP.},
- file = {:stutz2022learning - Learning Optimal Conformal Classifiers.pdf:PDF},
- month = may,
- year = {2022},
-}
-
-@InProceedings{grathwohl2020your,
- author = {Grathwohl, Will and Wang, Kuan-Chieh and Jacobsen, Joern-Henrik and Duvenaud, David and Norouzi, Mohammad and Swersky, Kevin},
- title = {Your classifier is secretly an energy based model and you should treat it like one},
- language = {en},
- url = {https://openreview.net/forum?id=Hkxzx0NtDB},
- urldate = {2023-02-13},
- abstract = {We propose to reinterpret a standard discriminative classifier of p(y{\textbar}x) as an energy based model for the joint distribution p(x, y). In this setting, the standard class probabilities can be easily computed as well as unnormalized values of p(x) and p(x{\textbar}y). Within this framework, standard discriminative architectures may be used and the model can also be trained on unlabeled data. We demonstrate that energy based training of the joint distribution improves calibration, robustness, and out-of-distribution detection while also enabling our models to generate samples rivaling the quality of recent GAN approaches. We improve upon recently proposed techniques for scaling up the training of energy based models and present an approach which adds little overhead compared to standard classification training. Our approach is the first to achieve performance rivaling the state-of-the-art in both generative and discriminative learning within one hybrid model.},
- file = {:grathwohl2020your - Your Classifier Is Secretly an Energy Based Model and You Should Treat It like One.pdf:PDF},
- month = mar,
- year = {2020},
-}
-
-@Book{murphy2023probabilistic,
- author = {Murphy, Kevin P.},
- date = {2023},
- title = {Probabilistic machine learning: {Advanced} topics},
- publisher = {MIT Press},
- shorttitle = {Probabilistic machine learning},
-}
-
-@TechReport{artelt2021evaluating,
- author = {Artelt, André and Vaquet, Valerie and Velioglu, Riza and Hinder, Fabian and Brinkrolf, Johannes and Schilling, Malte and Hammer, Barbara},
- date = {2021-07},
- institution = {arXiv},
- title = {Evaluating {Robustness} of {Counterfactual} {Explanations}},
- note = {arXiv:2103.02354 [cs] type: article},
- url = {http://arxiv.org/abs/2103.02354},
- urldate = {2023-03-24},
- abstract = {Transparency is a fundamental requirement for decision making systems when these should be deployed in the real world. It is usually achieved by providing explanations of the system's behavior. A prominent and intuitive type of explanations are counterfactual explanations. Counterfactual explanations explain a behavior to the user by proposing actions -- as changes to the input -- that would cause a different (specified) behavior of the system. However, such explanation methods can be unstable with respect to small changes to the input -- i.e. even a small change in the input can lead to huge or arbitrary changes in the output and of the explanation. This could be problematic for counterfactual explanations, as two similar individuals might get very different explanations. Even worse, if the recommended actions differ considerably in their complexity, one would consider such unstable (counterfactual) explanations as individually unfair. In this work, we formally and empirically study the robustness of counterfactual explanations in general, as well as under different models and different kinds of perturbations. Furthermore, we propose that plausible counterfactual explanations can be used instead of closest counterfactual explanations to improve the robustness and consequently the individual fairness of counterfactual explanations.},
- annotation = {Comment: Rewrite paper to make things more clear; Remove one theorem \& corollary due to buggy proof},
- file = {:artelt2021evaluating - Evaluating Robustness of Counterfactual Explanations.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
-}
-
-@Article{guidotti2022counterfactual,
- author = {Guidotti, Riccardo},
- date = {2022-04},
- journaltitle = {Data Mining and Knowledge Discovery},
- title = {Counterfactual explanations and how to find them: literature review and benchmarking},
- doi = {10.1007/s10618-022-00831-6},
- issn = {1573-756X},
- language = {en},
- url = {https://doi.org/10.1007/s10618-022-00831-6},
- urldate = {2023-03-24},
- abstract = {Interpretable machine learning aims at unveiling the reasons behind predictions returned by uninterpretable classifiers. One of the most valuable types of explanation consists of counterfactuals. A counterfactual explanation reveals what should have been different in an instance to observe a diverse outcome. For instance, a bank customer asks for a loan that is rejected. The counterfactual explanation consists of what should have been different for the customer in order to have the loan accepted. Recently, there has been an explosion of proposals for counterfactual explainers. The aim of this work is to survey the most recent explainers returning counterfactual explanations. We categorize explainers based on the approach adopted to return the counterfactuals, and we label them according to characteristics of the method and properties of the counterfactuals returned. In addition, we visually compare the explanations, and we report quantitative benchmarking assessing minimality, actionability, stability, diversity, discriminative power, and running time. The results make evident that the current state of the art does not provide a counterfactual explainer able to guarantee all these properties simultaneously.},
- file = {Full Text PDF:https\://link.springer.com/content/pdf/10.1007%2Fs10618-022-00831-6.pdf:application/pdf},
- keywords = {Explainable AI, Counterfactual explanations, Contrastive explanations, Interpretable machine learning},
- shorttitle = {Counterfactual explanations and how to find them},
-}
-
-@TechReport{mahajan2020preserving,
- author = {Mahajan, Divyat and Tan, Chenhao and Sharma, Amit},
- date = {2020-06},
- institution = {arXiv},
- title = {Preserving {Causal} {Constraints} in {Counterfactual} {Explanations} for {Machine} {Learning} {Classifiers}},
- doi = {10.48550/arXiv.1912.03277},
- note = {arXiv:1912.03277 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1912.03277},
- urldate = {2023-03-24},
- abstract = {To construct interpretable explanations that are consistent with the original ML model, counterfactual examples---showing how the model's output changes with small perturbations to the input---have been proposed. This paper extends the work in counterfactual explanations by addressing the challenge of feasibility of such examples. For explanations of ML models in critical domains such as healthcare and finance, counterfactual examples are useful for an end-user only to the extent that perturbation of feature inputs is feasible in the real world. We formulate the problem of feasibility as preserving causal relationships among input features and present a method that uses (partial) structural causal models to generate actionable counterfactuals. When feasibility constraints cannot be easily expressed, we consider an alternative mechanism where people can label generated CF examples on feasibility: whether it is feasible to intervene and realize the candidate CF example from the original input. To learn from this labelled feasibility data, we propose a modified variational auto encoder loss for generating CF examples that optimizes for feasibility as people interact with its output. Our experiments on Bayesian networks and the widely used ''Adult-Income'' dataset show that our proposed methods can generate counterfactual explanations that better satisfy feasibility constraints than existing methods.. Code repository can be accessed here: {\textbackslash}textit\{https://github.com/divyat09/cf-feasibility\}},
- annotation = {Comment: 2019 NeurIPS Workshop on Do the right thing: Machine learning and Causal Inference for improved decision making},
- file = {:mahajan2020preserving - Preserving Causal Constraints in Counterfactual Explanations for Machine Learning Classifiers.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Statistics - Machine Learning},
-}
-
-@TechReport{antoran2023sampling,
- author = {Antorán, Javier and Padhy, Shreyas and Barbano, Riccardo and Nalisnick, Eric and Janz, David and Hernández-Lobato, José Miguel},
- date = {2023-03},
- institution = {arXiv},
- title = {Sampling-based inference for large linear models, with application to linearised {Laplace}},
- note = {arXiv:2210.04994 [cs, stat] type: article},
- url = {http://arxiv.org/abs/2210.04994},
- urldate = {2023-03-25},
- abstract = {Large-scale linear models are ubiquitous throughout machine learning, with contemporary application as surrogate models for neural network uncertainty quantification; that is, the linearised Laplace method. Alas, the computational cost associated with Bayesian linear models constrains this method's application to small networks, small output spaces and small datasets. We address this limitation by introducing a scalable sample-based Bayesian inference method for conjugate Gaussian multi-output linear models, together with a matching method for hyperparameter (regularisation) selection. Furthermore, we use a classic feature normalisation method (the g-prior) to resolve a previously highlighted pathology of the linearised Laplace method. Together, these contributions allow us to perform linearised neural network inference with ResNet-18 on CIFAR100 (11M parameters, 100 outputs x 50k datapoints), with ResNet-50 on Imagenet (50M parameters, 1000 outputs x 1.2M datapoints) and with a U-Net on a high-resolution tomographic reconstruction task (2M parameters, 251k output{\textasciitilde}dimensions).},
- annotation = {Comment: Published at ICLR 2023. This latest Arxiv version is extended with a demonstration of the proposed methods on the Imagenet dataset},
- file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2210.04994.pdf:application/pdf},
- keywords = {Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
-}
-
-@Misc{altmeyer2022conformal,
- author = {Altmeyer, Patrick},
- date = {2022-10},
- title = {{Conformal} {Prediction} in {Julia}},
- language = {en},
- url = {https://www.paltmeyer.com/blog/posts/conformal-prediction/},
- urldate = {2023-03-27},
- abstract = {A (very) gentle introduction to Conformal Prediction in Julia using my new package ConformalPrediction.jl.},
-}
-
-@InProceedings{welling2011bayesian,
- author = {Welling, M. and Teh, Y.},
- date = {2011-06},
- title = {Bayesian {Learning} via {Stochastic} {Gradient} {Langevin} {Dynamics}},
- url = {https://www.semanticscholar.org/paper/Bayesian-Learning-via-Stochastic-Gradient-Langevin-Welling-Teh/aeed631d6a84100b5e9a021ec1914095c66de415},
- urldate = {2023-05-15},
- abstract = {In this paper we propose a new framework for learning from large scale datasets based on iterative learning from small mini-batches. By adding the right amount of noise to a standard stochastic gradient optimization algorithm we show that the iterates will converge to samples from the true posterior distribution as we anneal the stepsize. This seamless transition between optimization and Bayesian posterior sampling provides an inbuilt protection against overfitting. We also propose a practical method for Monte Carlo estimates of posterior statistics which monitors a "sampling threshold" and collects samples after it has been surpassed. We apply the method to three models: a mixture of Gaussians, logistic regression and ICA with natural gradients.},
- annotation = {[TLDR] This paper proposes a new framework for learning from large scale datasets based on iterative learning from small mini-batches by adding the right amount of noise to a standard stochastic gradient optimization algorithm and shows that the iterates will converge to samples from the true posterior distribution as the authors anneal the stepsize.},
- file = {:welling_bayesian_2011 - Bayesian Learning Via Stochastic Gradient Langevin Dynamics.html:URL;:welling2011bayesian - Bayesian Learning Via Stochastic Gradient Langevin Dynamics.pdf:PDF},
-}
-
-@Article{gill2010circular,
- author = {Gill, Jeff and Hangartner, Dominik},
- date = {2010},
- journaltitle = {Political Analysis},
- title = {Circular {Data} in {Political} {Science} and {How} to {Handle} {It}},
- doi = {10.1093/pan/mpq009},
- issn = {1047-1987, 1476-4989},
- language = {en},
- number = {3},
- pages = {316--336},
- url = {https://www.cambridge.org/core/journals/political-analysis/article/circular-data-in-political-science-and-how-to-handle-it/6DF2D9DA60C455E6A48FFB0FF011F747},
- urldate = {2023-05-15},
- volume = {18},
- abstract = {There has been no attention to circular (purely cyclical) data in political science research. We show that such data exist and are mishandled by models that do not take into account the inherently recycling nature of some phenomenon. Clock and calendar effects are the obvious cases, but directional data are observed as well. We describe a standard maximum likelihood regression modeling framework based on the von Mises distribution, then develop a general Bayesian regression procedure for the first time, providing an easy-to-use Metropolis-Hastings sampler for this approach. Applications include a chronographic analysis of U.S. domestic terrorism and directional party preferences in a two-dimensional ideological space for German Bundestag elections. The results demonstrate the importance of circular models to handle periodic and directional data in political science.},
- file = {Full Text PDF:https\://www.cambridge.org/core/services/aop-cambridge-core/content/view/6DF2D9DA60C455E6A48FFB0FF011F747/S1047198700012493a.pdf/div-class-title-circular-data-in-political-science-and-how-to-handle-it-div.pdf:application/pdf},
- publisher = {Cambridge University Press},
-}
-
-@InProceedings{liu2023goggle,
- author = {Liu, Tennison and Qian, Zhaozhi and Berrevoets, Jeroen and Schaar, Mihaela van der},
- date = {2023-02},
- title = {{GOGGLE}: {Generative} {Modelling} for {Tabular} {Data} by {Learning} {Relational} {Structure}},
- language = {en},
- url = {https://openreview.net/forum?id=fPVRcJqspu},
- urldate = {2023-05-15},
- abstract = {Deep generative models learn highly complex and non-linear representations to generate realistic synthetic data. While they have achieved notable success in computer vision and natural language processing, similar advances have been less demonstrable in the tabular domain. This is partially because generative modelling of tabular data entails a particular set of challenges, including heterogeneous relationships, limited number of samples, and difficulties in incorporating prior knowledge. Additionally, unlike their counterparts in image and sequence domain, deep generative models for tabular data almost exclusively employ fully-connected layers, which encode weak inductive biases about relationships between inputs. Real-world data generating processes can often be represented using relational structures, which encode sparse, heterogeneous relationships between variables. In this work, we learn and exploit relational structure underlying tabular data to better model variable dependence, and as a natural means to introduce regularization on relationships and include prior knowledge. Specifically, we introduce GOGGLE, an end-to-end message passing scheme that jointly learns the relational structure and corresponding functional relationships as the basis of generating synthetic samples. Using real-world datasets, we provide empirical evidence that the proposed method is effective in generating realistic synthetic data and exploiting domain knowledge for downstream tasks.},
- file = {Full Text PDF:https\://openreview.net/pdf?id=fPVRcJqspu:application/pdf},
- shorttitle = {{GOGGLE}},
-}
-
-@TechReport{du2020implicit,
- author = {Du, Yilun and Mordatch, Igor},
- date = {2020-06},
- institution = {arXiv},
- title = {Implicit {Generation} and {Generalization} in {Energy}-{Based} {Models}},
- doi = {10.48550/arXiv.1903.08689},
- note = {arXiv:1903.08689 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1903.08689},
- urldate = {2023-05-16},
- abstract = {Energy based models (EBMs) are appealing due to their generality and simplicity in likelihood modeling, but have been traditionally difficult to train. We present techniques to scale MCMC based EBM training on continuous neural networks, and we show its success on the high-dimensional data domains of ImageNet32x32, ImageNet128x128, CIFAR-10, and robotic hand trajectories, achieving better samples than other likelihood models and nearing the performance of contemporary GAN approaches, while covering all modes of the data. We highlight some unique capabilities of implicit generation such as compositionality and corrupt image reconstruction and inpainting. Finally, we show that EBMs are useful models across a wide variety of tasks, achieving state-of-the-art out-of-distribution classification, adversarially robust classification, state-of-the-art continual online class learning, and coherent long term predicted trajectory rollouts.},
- file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1903.08689.pdf:application/pdf},
- keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Statistics - Machine Learning},
-}
-
-@Comment{jabref-meta: databaseType:biblatex;}
diff --git a/experiments/Manifest.toml b/experiments/Manifest.toml
index 476b0179f5c28c654fbd17ccba65c1bcafd8f6b0..e39e6d988ae0e59747b64c5a7d557f835e996573 100644
--- a/experiments/Manifest.toml
+++ b/experiments/Manifest.toml
@@ -2,7 +2,7 @@
julia_version = "1.9.3"
manifest_format = "2.0"
-project_hash = "0a415d28608389ad9ae62c04d7b53f2cbcb4439a"
+project_hash = "4b0671d5fb3c16506a733dc9942e46bad62320c0"
[[deps.ARFFFiles]]
deps = ["CategoricalArrays", "Dates", "Parsers", "Tables"]
@@ -173,6 +173,12 @@ git-tree-sha1 = "4ae47f9a4b1dc19897d3743ff13685925c5202ec"
uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
version = "1.2.1"
+[[deps.Bzip2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
+uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
+version = "1.0.8+0"
+
[[deps.CEnum]]
git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
@@ -220,6 +226,12 @@ git-tree-sha1 = "75923dce4275ead3799b238e10178a68c07dbd3b"
uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645"
version = "8.9.4+0"
+[[deps.Cairo_jll]]
+deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2"
+uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
+version = "1.16.1+1"
+
[[deps.Calculus]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad"
@@ -409,6 +421,11 @@ git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc"
uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5"
version = "0.1.3"
+[[deps.Contour]]
+git-tree-sha1 = "d05d9e7b7aedff4e5b51a029dced05cfb6125781"
+uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
+version = "0.6.2"
+
[[deps.CoordinateTransformations]]
deps = ["LinearAlgebra", "StaticArrays"]
git-tree-sha1 = "f9d7112bfff8a19a3a4ea4e03a8e6a91fe8456bf"
@@ -571,6 +588,12 @@ git-tree-sha1 = "98fdf08b707aaf69f524a6cd0a67858cefe0cfb6"
uuid = "792122b4-ca99-40de-a6bc-6742525f08b6"
version = "0.3.0"
+[[deps.EpollShim_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "8e9441ee83492030ace98f9789a654a6d0b1f643"
+uuid = "2702e6a9-849d-5ed8-8c21-79e8b8f9ee43"
+version = "0.0.20230411+0"
+
[[deps.EvoTrees]]
deps = ["BSON", "CUDA", "CategoricalArrays", "Distributions", "MLJModelInterface", "NetworkLayout", "Random", "RecipesBase", "Statistics", "StatsBase", "Tables"]
git-tree-sha1 = "a1fa1d1743478394a0a7188d054b67546e4ca143"
@@ -583,6 +606,12 @@ git-tree-sha1 = "e90caa41f5a86296e014e148ee061bd6c3edec96"
uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4"
version = "0.1.9"
+[[deps.Expat_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c"
+uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
+version = "2.5.0+0"
+
[[deps.ExprTools]]
git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
@@ -593,6 +622,18 @@ git-tree-sha1 = "5e1e4c53fa39afe63a7d356e30452249365fba99"
uuid = "411431e0-e8b7-467b-b5e0-f676ba4f2910"
version = "0.1.1"
+[[deps.FFMPEG]]
+deps = ["FFMPEG_jll"]
+git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8"
+uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
+version = "0.4.1"
+
+[[deps.FFMPEG_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Pkg", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
+git-tree-sha1 = "74faea50c1d007c85837327f6775bea60b5492dd"
+uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
+version = "4.4.2+2"
+
[[deps.FFTViews]]
deps = ["CustomUnitRanges", "FFTW"]
git-tree-sha1 = "cbdf14d1e8c7c8aacbe8b19862e0179fd08321c2"
@@ -673,6 +714,18 @@ version = "0.14.6"
Metal = "dde4c033-4e86-420c-a63e-0dd931031962"
cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
+[[deps.Fontconfig_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03"
+uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
+version = "2.13.93+0"
+
+[[deps.Formatting]]
+deps = ["Printf"]
+git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
+uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
+version = "0.4.2"
+
[[deps.ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"]
git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad"
@@ -683,6 +736,18 @@ weakdeps = ["StaticArrays"]
[deps.ForwardDiff.extensions]
ForwardDiffStaticArraysExt = "StaticArrays"
+[[deps.FreeType2_jll]]
+deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"]
+git-tree-sha1 = "d8db6a5a2fe1381c1ea4ef2cab7c69c2de7f9ea0"
+uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
+version = "2.13.1+0"
+
+[[deps.FriBidi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91"
+uuid = "559328eb-81f9-559d-9380-de523a88c83c"
+version = "1.0.10+0"
+
[[deps.Functors]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "9a68d75d466ccc1218d0552a8e1631151c569545"
@@ -693,6 +758,12 @@ version = "0.4.5"
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
+[[deps.GLFW_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
+git-tree-sha1 = "d972031d28c8c8d9d7b41a536ad7bb0c2579caca"
+uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
+version = "3.3.8+0"
+
[[deps.GPUArrays]]
deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"]
git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1"
@@ -711,6 +782,18 @@ git-tree-sha1 = "72b2e3c2ba583d1a7aa35129e56cf92e07c083e3"
uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
version = "0.21.4"
+[[deps.GR]]
+deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Preferences", "Printf", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "UUIDs", "p7zip_jll"]
+git-tree-sha1 = "8e2d86e06ceb4580110d9e716be26658effc5bfd"
+uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
+version = "0.72.8"
+
+[[deps.GR_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "da121cbdc95b065da07fbb93638367737969693f"
+uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
+version = "0.72.8+0"
+
[[deps.GZip]]
deps = ["Libdl", "Zlib_jll"]
git-tree-sha1 = "6388a2d8e409ce23de7d03a7c73d83c5753b3eb2"
@@ -729,6 +812,18 @@ git-tree-sha1 = "424a5a6ce7c5d97cca7bcc4eac551b97294c54af"
uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
version = "0.4.9"
+[[deps.Gettext_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046"
+uuid = "78b55507-aeef-58d4-861c-77aaff3498b1"
+version = "0.21.0+0"
+
+[[deps.Glib_jll]]
+deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"]
+git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17"
+uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
+version = "2.76.5+0"
+
[[deps.Glob]]
git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496"
uuid = "c27321d9-0574-5035-807b-f59d2c89b15c"
@@ -740,12 +835,23 @@ git-tree-sha1 = "d61890399bc535850c4bf08e4e0d3a7ad0f21cbd"
uuid = "a2bd30eb-e257-5431-a919-1863eab51364"
version = "1.1.2"
+[[deps.Graphite2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011"
+uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472"
+version = "1.3.14+0"
+
[[deps.Graphs]]
deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"]
git-tree-sha1 = "1cf1d7dcb4bc32d7b4a5add4232db3750c27ecb4"
uuid = "86223c79-3864-5bf0-83f7-82e725a168b6"
version = "1.8.0"
+[[deps.Grisu]]
+git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
+uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe"
+version = "1.0.2"
+
[[deps.HDF5]]
deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Printf", "Random", "Requires", "UUIDs"]
git-tree-sha1 = "114e20044677badbc631ee6fdc80a67920561a29"
@@ -753,10 +859,10 @@ uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
version = "0.16.16"
[[deps.HDF5_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"]
-git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739"
+deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "4cc2bb72df6ff40b055295fdef6d92955f9dede8"
uuid = "0234f1f7-429e-5d53-9886-15a909be8d59"
-version = "1.14.2+1"
+version = "1.12.2+2"
[[deps.HTTP]]
deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
@@ -764,6 +870,12 @@ git-tree-sha1 = "5eab648309e2e060198b45820af1a37182de3cce"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "1.10.0"
+[[deps.HarfBuzz_jll]]
+deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
+git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3"
+uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
+version = "2.8.1+1"
+
[[deps.Highlights]]
deps = ["DocStringExtensions", "InteractiveUtils", "REPL"]
git-tree-sha1 = "0341077e8a6b9fc1c2ea5edc1e93a956d2aec0c7"
@@ -1004,6 +1116,12 @@ git-tree-sha1 = "c11d691a0dc8e90acfa4740d293ade57f68bfdbb"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.4.35"
+[[deps.JLFzf]]
+deps = ["Pipe", "REPL", "Random", "fzf_jll"]
+git-tree-sha1 = "f377670cda23b6b7c1c0b3893e37451c5c1a2185"
+uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c"
+version = "0.1.5"
+
[[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
@@ -1058,6 +1176,12 @@ version = "0.9.8"
[deps.KernelAbstractions.weakdeps]
EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
+[[deps.LAME_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c"
+uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
+version = "3.100.1+0"
+
[[deps.LERC_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "bf36f528eec6634efc60d7ec062008f171071434"
@@ -1082,6 +1206,12 @@ git-tree-sha1 = "f689897ccbe049adb19a065c495e75f372ecd42b"
uuid = "1d63c593-3942-5779-bab2-d838dc0a180e"
version = "15.0.4+0"
+[[deps.LZO_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
+uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
+version = "2.10.1+0"
+
[[deps.LaTeXStrings]]
git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996"
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
@@ -1093,6 +1223,20 @@ git-tree-sha1 = "ca7a96bd2be5066bb2378b42c0191c672811bfaa"
uuid = "c52c1a26-f7c5-402b-80be-ba1e638ad478"
version = "0.1.3"
+[[deps.Latexify]]
+deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Printf", "Requires"]
+git-tree-sha1 = "f428ae552340899a935973270b8d98e5a31c49fe"
+uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
+version = "0.16.1"
+
+ [deps.Latexify.extensions]
+ DataFramesExt = "DataFrames"
+ SymEngineExt = "SymEngine"
+
+ [deps.Latexify.weakdeps]
+ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+ SymEngine = "123dc426-2d89-5057-bbad-38513e3affd8"
+
[[deps.LatinHypercubeSampling]]
deps = ["Random", "StableRNGs", "StatsBase", "Test"]
git-tree-sha1 = "825289d43c753c7f1bf9bed334c253e9913997f8"
@@ -1136,18 +1280,54 @@ version = "1.10.2+0"
[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+[[deps.Libffi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290"
+uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
+version = "3.2.2+1"
+
+[[deps.Libgcrypt_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
+git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae"
+uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4"
+version = "1.8.7+0"
+
+[[deps.Libglvnd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"]
+git-tree-sha1 = "6f73d1dd803986947b2c750138528a999a6c7733"
+uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29"
+version = "1.6.0+0"
+
+[[deps.Libgpg_error_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9"
+uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8"
+version = "1.42.0+0"
+
[[deps.Libiconv_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.17.0+0"
+[[deps.Libmount_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
+uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
+version = "2.35.0+0"
+
[[deps.Libtiff_jll]]
deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"]
git-tree-sha1 = "3eb79b0ca5764d4799c06699573fd8f533259713"
uuid = "89763e89-9b03-5906-acba-b20f662cd828"
version = "4.4.0+0"
+[[deps.Libuuid_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066"
+uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
+version = "2.36.0+0"
+
[[deps.LinearAlgebra]]
deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -1356,6 +1536,11 @@ deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.28.2+0"
+[[deps.Measures]]
+git-tree-sha1 = "c13304c81eec1ed3af7fc20e75fb6b26092a1102"
+uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e"
+version = "0.3.2"
+
[[deps.MetaGraphs]]
deps = ["Graphs", "JLD2", "Random"]
git-tree-sha1 = "1130dbe1d5276cb656f6e1094ce97466ed700e5a"
@@ -1495,6 +1680,12 @@ git-tree-sha1 = "2ac17d29c523ce1cd38e27785a7d23024853a4bb"
uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
version = "1.12.10"
+[[deps.Ogg_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f"
+uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
+version = "1.3.5+1"
+
[[deps.OneHotArrays]]
deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"]
git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c"
@@ -1543,9 +1734,9 @@ version = "1.4.1"
[[deps.OpenSSL_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "ceeda72c9fd6bbebc4f4f598560789145a8b6c4c"
+git-tree-sha1 = "a12e56c72edee3ce6b96667745e6cbbe5498f200"
uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
-version = "3.0.11+0"
+version = "1.1.23+0"
[[deps.OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
@@ -1559,11 +1750,22 @@ git-tree-sha1 = "34205b1204cc83c43cd9cfe53ffbd3b310f6e8c5"
uuid = "3bd65402-5787-11e9-1adc-39752487f4e2"
version = "0.3.1"
+[[deps.Opus_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720"
+uuid = "91d4177d-7536-5919-b921-800302f37372"
+version = "1.3.2+0"
+
[[deps.OrderedCollections]]
git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.6.2"
+[[deps.PCRE2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15"
+version = "10.42.0+0"
+
[[deps.PDMats]]
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
git-tree-sha1 = "3129380a93388e5062e946974246fe3f2e7c73e2"
@@ -1617,6 +1819,17 @@ git-tree-sha1 = "2e71d7dbcab8dc47306c0ed6ac6018fbc1a7070f"
uuid = "fbb45041-c46e-462f-888f-7c521cafbc2c"
version = "0.3.3"
+[[deps.Pipe]]
+git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d"
+uuid = "b98c9c47-44ae-5843-9183-064241ee97a0"
+version = "1.3.0"
+
+[[deps.Pixman_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"]
+git-tree-sha1 = "64779bc4c9784fee475689a1752ef4d5747c5e87"
+uuid = "30392449-352a-5448-841d-b1acce4e97dc"
+version = "0.42.2+0"
+
[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -1634,6 +1847,38 @@ git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da"
uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688"
version = "0.3.3"
+[[deps.PlotThemes]]
+deps = ["PlotUtils", "Statistics"]
+git-tree-sha1 = "1f03a2d339f42dca4a4da149c7e15e9b896ad899"
+uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
+version = "3.1.0"
+
+[[deps.PlotUtils]]
+deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "Statistics"]
+git-tree-sha1 = "f92e1315dadf8c46561fb9396e525f7200cdc227"
+uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
+version = "1.3.5"
+
+[[deps.Plots]]
+deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Preferences", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "UnitfulLatexify", "Unzip"]
+git-tree-sha1 = "ccee59c6e48e6f2edf8a5b64dc817b6729f99eb5"
+uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+version = "1.39.0"
+
+ [deps.Plots.extensions]
+ FileIOExt = "FileIO"
+ GeometryBasicsExt = "GeometryBasics"
+ IJuliaExt = "IJulia"
+ ImageInTerminalExt = "ImageInTerminal"
+ UnitfulExt = "Unitful"
+
+ [deps.Plots.weakdeps]
+ FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+ GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+ IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a"
+ ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254"
+ Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
+
[[deps.PolyesterWeave]]
deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"]
git-tree-sha1 = "240d7170f5ffdb285f9427b92333c3463bf65bf6"
@@ -1712,11 +1957,17 @@ git-tree-sha1 = "18e8f4d1426e965c7b532ddd260599e1510d26ce"
uuid = "4b34888f-f399-49d4-9bb3-47ed5cae4e65"
version = "1.0.0"
+[[deps.Qt5Base_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"]
+git-tree-sha1 = "0c03844e2231e12fda4d0086fd7cbe4098ee8dc5"
+uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1"
+version = "5.15.3+2"
+
[[deps.QuadGK]]
deps = ["DataStructures", "LinearAlgebra"]
-git-tree-sha1 = "eeab25344bf9901146c0200a7ca64ea479f8bf5c"
+git-tree-sha1 = "9ebcd48c498668c7fa0e97a9cae873fbee7bfee1"
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
-version = "2.9.0"
+version = "2.9.1"
[[deps.Quaternions]]
deps = ["LinearAlgebra", "Random", "RealDot"]
@@ -1771,6 +2022,12 @@ git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.3.4"
+[[deps.RecipesPipeline]]
+deps = ["Dates", "NaNMath", "PlotUtils", "PrecompileTools", "RecipesBase"]
+git-tree-sha1 = "45cf9fd0ca5839d06ef333c8201714e888486342"
+uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
+version = "0.6.12"
+
[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
@@ -1874,6 +2131,12 @@ git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5"
uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3"
version = "0.1.0"
+[[deps.Showoff]]
+deps = ["Dates", "Grisu"]
+git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
+uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
+version = "1.0.3"
+
[[deps.SimpleBufferStream]]
git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"
uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7"
@@ -1969,9 +2232,9 @@ weakdeps = ["OffsetArrays", "StaticArrays"]
[[deps.StaticArrays]]
deps = ["LinearAlgebra", "Random", "StaticArraysCore"]
-git-tree-sha1 = "51621cca8651d9e334a659443a74ce50a3b6dfab"
+git-tree-sha1 = "d5fb407ec3179063214bc6277712928ba78459e2"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
-version = "1.6.3"
+version = "1.6.4"
weakdeps = ["Statistics"]
[deps.StaticArrays.extensions]
@@ -2200,6 +2463,12 @@ git-tree-sha1 = "903be579194534af1c4b4778d1ace676ca042238"
uuid = "a7773ee8-282e-5fa2-be4e-bd808c38a91a"
version = "1.0.0"
+[[deps.UnitfulLatexify]]
+deps = ["LaTeXStrings", "Latexify", "Unitful"]
+git-tree-sha1 = "e2d817cc500e960fdbafcf988ac8436ba3208bfd"
+uuid = "45397f5d-5981-4c77-b2b3-fc36d6e9b728"
+version = "1.6.3"
+
[[deps.UnsafeAtomics]]
git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278"
uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f"
@@ -2211,12 +2480,29 @@ git-tree-sha1 = "323e3d0acf5e78a56dfae7bd8928c989b4f3083e"
uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249"
version = "0.1.3"
+[[deps.Unzip]]
+git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78"
+uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d"
+version = "0.2.0"
+
[[deps.VectorizationBase]]
deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"]
git-tree-sha1 = "b182207d4af54ac64cbc71797765068fdeff475d"
uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
version = "0.21.64"
+[[deps.Wayland_jll]]
+deps = ["Artifacts", "EpollShim_jll", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "7558e29847e99bc3f04d6569e82d0f5c54460703"
+uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
+version = "1.21.0+1"
+
+[[deps.Wayland_protocols_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4528479aa01ee1b3b4cd0e6faef0e04cf16466da"
+uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
+version = "1.25.0+0"
+
[[deps.WeakRefStrings]]
deps = ["DataAPI", "InlineStrings", "Parsers"]
git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23"
@@ -2234,6 +2520,144 @@ git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7"
uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60"
version = "1.6.1"
+[[deps.XML2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"]
+git-tree-sha1 = "04a51d15436a572301b5abbb9d099713327e9fc4"
+uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
+version = "2.10.4+0"
+
+[[deps.XSLT_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"]
+git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a"
+uuid = "aed1982a-8fda-507f-9586-7b0439959a61"
+version = "1.1.34+0"
+
+[[deps.Xorg_libX11_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"]
+git-tree-sha1 = "afead5aba5aa507ad5a3bf01f58f82c8d1403495"
+uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc"
+version = "1.8.6+0"
+
+[[deps.Xorg_libXau_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "6035850dcc70518ca32f012e46015b9beeda49d8"
+uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec"
+version = "1.0.11+0"
+
+[[deps.Xorg_libXcursor_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd"
+uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724"
+version = "1.2.0+4"
+
+[[deps.Xorg_libXdmcp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "34d526d318358a859d7de23da945578e8e8727b7"
+uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05"
+version = "1.1.4+0"
+
+[[deps.Xorg_libXext_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3"
+uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3"
+version = "1.3.4+4"
+
+[[deps.Xorg_libXfixes_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4"
+uuid = "d091e8ba-531a-589c-9de9-94069b037ed8"
+version = "5.0.3+4"
+
+[[deps.Xorg_libXi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"]
+git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246"
+uuid = "a51aa0fd-4e3c-5386-b890-e753decda492"
+version = "1.7.10+4"
+
+[[deps.Xorg_libXinerama_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"]
+git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123"
+uuid = "d1454406-59df-5ea1-beac-c340f2130bc3"
+version = "1.1.4+4"
+
+[[deps.Xorg_libXrandr_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631"
+uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484"
+version = "1.5.2+4"
+
+[[deps.Xorg_libXrender_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96"
+uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa"
+version = "0.9.10+4"
+
+[[deps.Xorg_libpthread_stubs_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "8fdda4c692503d44d04a0603d9ac0982054635f9"
+uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74"
+version = "0.1.1+0"
+
+[[deps.Xorg_libxcb_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"]
+git-tree-sha1 = "b4bfde5d5b652e22b9c790ad00af08b6d042b97d"
+uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b"
+version = "1.15.0+0"
+
+[[deps.Xorg_libxkbfile_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"]
+git-tree-sha1 = "730eeca102434283c50ccf7d1ecdadf521a765a4"
+uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a"
+version = "1.1.2+0"
+
+[[deps.Xorg_xcb_util_image_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97"
+uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b"
+version = "0.4.0+1"
+
+[[deps.Xorg_xcb_util_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"]
+git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1"
+uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5"
+version = "0.4.0+1"
+
+[[deps.Xorg_xcb_util_keysyms_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00"
+uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7"
+version = "0.4.0+1"
+
+[[deps.Xorg_xcb_util_renderutil_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e"
+uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e"
+version = "0.3.9+1"
+
+[[deps.Xorg_xcb_util_wm_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67"
+uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361"
+version = "0.4.1+1"
+
+[[deps.Xorg_xkbcomp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxkbfile_jll"]
+git-tree-sha1 = "330f955bc41bb8f5270a369c473fc4a5a4e4d3cb"
+uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4"
+version = "1.4.6+0"
+
+[[deps.Xorg_xkeyboard_config_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xkbcomp_jll"]
+git-tree-sha1 = "691634e5453ad362044e2ad653e79f3ee3bb98c3"
+uuid = "33bec58e-1273-512f-9401-5d533626f822"
+version = "2.39.0+0"
+
+[[deps.Xorg_xtrans_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "e92a1a012a10506618f10b7047e478403a046c77"
+uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10"
+version = "1.5.0+0"
+
[[deps.ZipFile]]
deps = ["Libdl", "Printf", "Zlib_jll"]
git-tree-sha1 = "f492b7fe1698e623024e873244f10d89c95c340a"
@@ -2279,23 +2703,41 @@ git-tree-sha1 = "5a1ba43303c62f4a09b0d6751422de03424ab0cd"
uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
version = "1.1.1"
+[[deps.fzf_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "868e669ccb12ba16eaf50cb2957ee2ff61261c56"
+uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09"
+version = "0.29.0+0"
+
[[deps.ghr_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "a83b3feeda837dd3f3cad19076bda0f0a524d687"
uuid = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
version = "0.14.0+0"
-[[deps.libaec_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "eddd19a8dea6b139ea97bdc8a0e2667d4b661720"
-uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0"
-version = "1.0.6+1"
+[[deps.libaom_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "3a2ea60308f0996d26f1e5354e10c24e9ef905d4"
+uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b"
+version = "3.4.0+0"
+
+[[deps.libass_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
+uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
+version = "0.15.1+0"
[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
version = "5.8.0+0"
+[[deps.libfdk_aac_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
+uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
+version = "2.0.2+0"
+
[[deps.libpng_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c"
@@ -2308,6 +2750,12 @@ git-tree-sha1 = "d4f63314c8aa1e48cd22aa0c17ed76cd1ae48c3c"
uuid = "075b6546-f08a-558a-be8f-8157d0f608a5"
version = "1.10.3+0"
+[[deps.libvorbis_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
+git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c"
+uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
+version = "1.3.7+1"
+
[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
@@ -2317,3 +2765,21 @@ version = "1.48.0+0"
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
version = "17.4.0+0"
+
+[[deps.x264_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2"
+uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
+version = "2021.5.5+0"
+
+[[deps.x265_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9"
+uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
+version = "3.5.0+0"
+
+[[deps.xkbcommon_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
+git-tree-sha1 = "9c304562909ab2bab0262639bd4f444d7bc2be37"
+uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd"
+version = "1.4.1+1"
diff --git a/experiments/Project.toml b/experiments/Project.toml
index ed2f33f80a6da7c620763050dace63f7bc5e9d33..6713ecf6dd4e38c674f4d2fafa22e8c28492709b 100644
--- a/experiments/Project.toml
+++ b/experiments/Project.toml
@@ -20,7 +20,9 @@ MLJFlux = "094fc8d1-fd35-5302-93ea-dabda2abf845"
MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
Metalhead = "dbeba491-748d-5e0e-a39e-b530a07fa0cc"
+MosaicViews = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389"
MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
diff --git a/experiments/grid_search.jl b/experiments/grid_search.jl
index 32a06eb32c9c816152f498e134dc3dbb21b54355..323c47c94f663e264919eb1f17c7809a1b95a994 100644
--- a/experiments/grid_search.jl
+++ b/experiments/grid_search.jl
@@ -71,6 +71,7 @@ const ECCCo_Δ_NAMES = [
"ECCCo-Δ",
"ECCCo-Δ (no CP)",
"ECCCo-Δ (no EBM)",
+ "ECCCo-Δ (latent)",
]
"""
@@ -117,7 +118,7 @@ Return the best outcome from grid search results. The best outcome is defined as
function best_absolute_outcome(
outcomes::Dict;
generator=ECCCO_NAMES,
- measure::AbstractArray=["distance_from_targets_l2", "distance_from_energy_l2"],
+ measure::AbstractArray=["distance_from_energy_l2"],
model::Union{Nothing,AbstractArray}=nothing,
weights::Union{Nothing,AbstractArray}=nothing
)
@@ -129,7 +130,7 @@ function best_absolute_outcome(
for (params, outcome) in outcomes
# Setup
- evaluation = outcome.bmk.evaluation
+ evaluation = deepcopy(outcome.bmk.evaluation)
exper = outcome.exper
generator_dict = outcome.generator_dict
model_dict = outcome.model_dict
diff --git a/experiments/jobscripts/tuning/generators/california_housing.sh b/experiments/jobscripts/tuning/generators/california_housing.sh
index 09c7c1e2625f15fda6225612da368501b536d425..373d660794d6c109859adf8a88972d01589574d6 100644
--- a/experiments/jobscripts/tuning/generators/california_housing.sh
+++ b/experiments/jobscripts/tuning/generators/california_housing.sh
@@ -11,4 +11,4 @@
module load 2023r1 openmpi
-srun julia --project=experiments experiments/run_experiments.jl -- data=california_housing output_path=results mpi grid_search n_individuals=25 > experiments/grid_search_california_housing.log
+srun julia --project=experiments experiments/run_experiments.jl -- data=california_housing output_path=results mpi grid_search n_individuals=25 store_ce > experiments/grid_search_california_housing.log
diff --git a/experiments/jobscripts/tuning/generators/tabular.sh b/experiments/jobscripts/tuning/generators/tabular.sh
index 9b73897c46de9fa8bf74680d6616ebc586994084..3c043ea7ea9d486169a8fcadb3849be59ccfb7e4 100644
--- a/experiments/jobscripts/tuning/generators/tabular.sh
+++ b/experiments/jobscripts/tuning/generators/tabular.sh
@@ -11,4 +11,4 @@
module load 2023r1 openmpi
-srun julia --project=experiments experiments/run_experiments.jl -- data=gmsc,german_credit output_path=results mpi grid_search n_individuals=25 > experiments/grid_search_tabular.log
+srun julia --project=experiments experiments/run_experiments.jl -- data=gmsc,german_credit output_path=results mpi grid_search n_individuals=25 store_ce > experiments/grid_search_tabular.log
diff --git a/experiments/post_processing/plotting.jl b/experiments/post_processing/plotting.jl
new file mode 100644
index 0000000000000000000000000000000000000000..46d36c76a382cc41db7f384b1804ac30b8f467d8
--- /dev/null
+++ b/experiments/post_processing/plotting.jl
@@ -0,0 +1,204 @@
+using Plots
+
+function choose_random_mnist(outcome::ExperimentOutcome; model::String="LeNet-5", img_height=125, seed=966)
+
+ # Set seed:
+ if !isnothing(seed)
+ Random.seed!(seed)
+ end
+
+ # Get output:
+ bmk = outcome.bmk()
+ grouped_bmk = groupby(bmk[bmk.variable.=="distance" .&& bmk.model.==model,:], [:dataname, :target, :factual])
+ random_choice = rand(1:length(grouped_bmk))
+ generators = unique(bmk.generator)
+ n_generators = length(generators)
+
+ # Get data:
+ df = grouped_bmk[random_choice][1:n_generators, :] |>
+ x -> sort(x, :generator) |>
+ x -> subset(x, :generator => ByRow(x -> x != "ECCCo"))
+ generators = df.generator
+ replace!(generators, "ECCCo-Δ" => "ECCCo")
+ replace!(generators, "ECCCo-Δ (latent)" => "ECCCo+")
+ n_generators = length(generators)
+
+ # Factual:
+ img = CounterfactualExplanations.factual(grouped_bmk[random_choice][1:n_generators,:].ce[1]) |> ECCCo.convert2mnist
+ p1 = Plots.plot(
+ img,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="Factual",
+ )
+ plts = [p1]
+ ces = []
+
+ # Counterfactuals:
+ for (i, generator) in enumerate(generators)
+ ce = df.ce[i]
+ img = CounterfactualExplanations.counterfactual(ce) |> ECCCo.convert2mnist
+ p = Plots.plot(
+ img,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="$generator",
+ )
+ push!(plts, p)
+ push!(ces, ce)
+ end
+
+ plt = Plots.plot(
+ plts...,
+ layout=(1, n_generators + 1),
+ size=(img_height * (n_generators + 1), img_height),
+ dpi=300
+ )
+ display(plt)
+
+ return plt, df.target[1], seed, ces, df.sample[1]
+
+end
+
+function plot_random_eccco(outcome::ExperimentOutcome; ce=nothing, generator="ECCCo-Δ", img_height=200, seed=966)
+ # Set seed:
+ if !isnothing(seed)
+ Random.seed!(seed)
+ end
+
+ # Get output:
+ bmk = outcome.bmk()
+ ce = isnothing(ce) ? rand(bmk.ce) : ce
+ gen = outcome.generator_dict[generator]
+ models = outcome.model_dict
+ x = CounterfactualExplanations.counterfactual(ce)
+ target = ce.target
+ data = ce.data
+
+ # Factual:
+ img = CounterfactualExplanations.factual(ce) |> ECCCo.convert2mnist
+ p1 = Plots.plot(
+ img,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="Factual",
+ )
+ plts = [p1]
+
+ for (model_name, M) in models
+ ce = generate_counterfactual(x, target, data, M, gen; initialization=:identity, converge_when=:generator_conditions)
+ img = CounterfactualExplanations.counterfactual(ce) |> ECCCo.convert2mnist
+ p = Plots.plot(
+ img,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="$model_name",
+ )
+ push!(plts, p)
+ end
+ n_models = length(models)
+
+ plt = Plots.plot(
+ plts...,
+ layout=(1, n_models + 1),
+ size=(img_height * (n_models + 1), img_height),
+ dpi=300
+ )
+ display(plt)
+
+ return plt, target, seed
+end
+
+function plot_all_mnist(gen, model, data=load_mnist_test(); img_height=150, seed=123, maxoutdim=64)
+
+ # Set seed:
+ if !isnothing(seed)
+ Random.seed!(seed)
+ end
+
+ # Dimensionality reduction:
+ data.dt = MultivariateStats.fit(MultivariateStats.PCA, data.X; maxoutdim=maxoutdim)
+
+ # VAE for REVISE:
+ data.generative_model = CounterfactualExplanations.Models.load_mnist_vae()
+
+ targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ factuals = targets
+ plts = []
+
+ for factual in factuals
+ chosen = rand(findall(data.output_encoder.labels .== factual))
+ x = select_factual(data, chosen)
+ for target in targets
+ if factual != target
+ @info "Generating counterfactual for $(factual) -> $(target)"
+ ce = generate_counterfactual(
+ x, target, data, model, gen;
+ initialization=:identity, converge_when=:generator_conditions
+ )
+ plt = Plots.plot(
+ CounterfactualExplanations.counterfactual(ce) |> ECCCo.convert2mnist,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="$factual → $target",
+ )
+ else
+ plt = Plots.plot(
+ x |> ECCCo.convert2mnist,
+ axis=([], false),
+ size=(img_height, img_height),
+ title="Factual",
+ )
+ end
+ push!(plts, plt)
+ end
+ end
+
+ plt = Plots.plot(
+ plts...,
+ layout=(length(factuals), length(targets)),
+ size=(img_height * length(targets), img_height * length(factuals)),
+ dpi=300
+ )
+
+ return plt
+
+end
+
+using MLDatasets
+using MosaicViews
+function vae_reconstructions(seed=123)
+
+ # Set seed:
+ if !isnothing(seed)
+ Random.seed!(seed)
+ end
+
+ counterfactual_data = load_mnist()
+ counterfactual_data.generative_model = CounterfactualExplanations.Models.load_mnist_vae()
+ X = counterfactual_data.X
+ y = counterfactual_data.output_encoder.y
+ images = []
+ rec_images = []
+ for i in 0:9
+ j = 0
+ while j < 10
+ x = X[:,rand(findall(y .== i))]
+ x̂ = CounterfactualExplanations.GenerativeModels.reconstruct(vae, x)[1] |>
+ x̂ -> clamp.((x̂ .+ 1.0) ./ 2.0, 0.0, 1.0) |>
+ x̂ -> reshape(x̂, 28,28) |>
+ x̂ -> MLDatasets.convert2image(MNIST, x̂)
+ x = clamp.((x .+ 1.0) ./ 2.0, 0.0, 1.0) |>
+ x -> reshape(x, 28,28) |>
+ x -> MLDatasets.convert2image(MNIST, x)
+ push!(images, x)
+ push!(rec_images, x̂)
+ j += 1
+ end
+ end
+ p1 = plot(mosaic(images..., ncol=10), title="Images")
+ p2 = plot(mosaic(rec_images..., ncol=10), title="Reconstructions")
+ plt = plot(p1, p2, axis=false, size=(800,375))
+
+ return plt
+end
\ No newline at end of file
diff --git a/experiments/post_processing/post_processing.jl b/experiments/post_processing/post_processing.jl
index f70aabe43c34d7474afc522dd98013210eab5543..a79b23b0fffa0ae154a6760b67a3eb9223e3018e 100644
--- a/experiments/post_processing/post_processing.jl
+++ b/experiments/post_processing/post_processing.jl
@@ -1,3 +1,4 @@
include("meta_data.jl")
include("artifacts.jl")
-include("results.jl")
\ No newline at end of file
+include("results.jl")
+include("plotting.jl")
\ No newline at end of file
diff --git a/experiments/setup_env.jl b/experiments/setup_env.jl
index 72e8f2ceb459dbacd2a7c3ca962fb0fd742baaba..4360454678cd075f1bd48437b8800acca81734f0 100644
--- a/experiments/setup_env.jl
+++ b/experiments/setup_env.jl
@@ -173,8 +173,8 @@ DEFAULT_GENERATOR_TUNING_LARGE = (
],
reg_strength=[0.0, 0.1, 0.5],
opt=[
- Flux.Optimise.Descent(0.05),
Flux.Optimise.Descent(0.01),
+ Flux.Optimise.Descent(0.05),
],
)
diff --git a/notebooks/tables.Rmd b/notebooks/tables.Rmd
index 1480ea7db975bc73d20ad5fd33a8e1fda4d25d7e..d41db45a03ea6cc5da9cae2701c23ee7aa258059 100644
--- a/notebooks/tables.Rmd
+++ b/notebooks/tables.Rmd
@@ -176,7 +176,7 @@ kbl(
format="latex", linesep = line_sep
) %>%
kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
add_header_above(header) %>%
collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
save_kable(file_name)
@@ -227,7 +227,7 @@ kbl(
format="latex", linesep = line_sep
) %>%
kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
add_header_above(header) %>%
collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
save_kable(file_name)
@@ -255,7 +255,7 @@ kbl(
format="latex"
) %>%
kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
collapse_rows(columns = 1:3, latex_hline = "custom", valign = "top", custom_latex_hline = 1:2) %>%
save_kable("paper/contents/table_all.tex")
```
@@ -282,7 +282,7 @@ kbl(
format="latex"
) %>%
kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
collapse_rows(columns = 1:3, latex_hline = "custom", valign = "top", custom_latex_hline = 1:2) %>%
save_kable("paper/contents/table_all_valid.tex")
```
@@ -317,7 +317,7 @@ kbl(
format="latex"
) %>%
kable_styling(font_size = 8) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
save_kable("paper/contents/table_ebm_params.tex")
```
@@ -337,7 +337,7 @@ kbl(
format="latex"
) %>%
kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
add_header_above(header) %>%
save_kable("paper/contents/table_params.tex")
```
@@ -361,7 +361,7 @@ kbl(
format="latex"
) %>%
kable_styling(font_size = 8) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
save_kable("paper/contents/table_gen_params.tex")
```
@@ -387,7 +387,7 @@ kbl(
format="latex", digits=2
) %>%
kable_styling(font_size = 8) %>%
- kable_paper(full_width = F) %>%
+ kable_paper(full_width = T) %>%
add_header_above(c(" "=2, "Performance Metrics" = 3)) %>%
collapse_rows(columns = 1, latex_hline = "custom", valign = "top", custom_latex_hline = 1) %>%
save_kable("paper/contents/table_perf.tex")
diff --git a/paper/.gitignore b/paper/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..075b2542afb820ca0c990f02a196dfbb35c41a3a
--- /dev/null
+++ b/paper/.gitignore
@@ -0,0 +1 @@
+/.quarto/
diff --git a/paper/_quarto.yml b/paper/_quarto.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a613903a22850342a682ac6fc84dd211a8536f6a
--- /dev/null
+++ b/paper/_quarto.yml
@@ -0,0 +1 @@
+bibliography: bib.bib
\ No newline at end of file
diff --git a/paper/aaai/aaai24.bst b/paper/aaai/aaai24.bst
new file mode 100644
index 0000000000000000000000000000000000000000..05b1d4e4414648ff46e9199f03bc66c9f77dedb5
--- /dev/null
+++ b/paper/aaai/aaai24.bst
@@ -0,0 +1,1493 @@
+%%
+%% This is file `aaai22.bst',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% merlin.mbs (with options: `head,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}')
+%% merlin.mbs (with options: `tail,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}')
+%% ----------------------------------------
+%% *** Natbib-compatible implementation of 'aaai' bib style ***
+%%
+ % ===============================================================
+ % IMPORTANT NOTICE:
+ % This bibliographic style (bst) file has been generated from one or
+ % more master bibliographic style (mbs) files, listed above.
+ %
+ % This generated file can be redistributed and/or modified under the terms
+ % of the LaTeX Project Public License Distributed from CTAN
+ % archives in directory macros/latex/base/lppl.txt; either
+ % version 1 of the License, or any later version.
+ % ===============================================================
+ % Name and version information of the main mbs file:
+ % \ProvidesFile{merlin.mbs}[2011/11/18 4.33 (PWD, AO, DPC)]
+ % For use with BibTeX version 0.99a or later
+ %-------------------------------------------------------------------
+ % This bibliography style file is intended for texts in ENGLISH
+ % This is an author-year citation style bibliography. As such, it is
+ % non-standard LaTeX, and requires a special package file to function properly.
+ % Such a package is natbib.sty by Patrick W. Daly
+ % The form of the \bibitem entries is
+ % \bibitem[Jones et al.(1990)]{key}...
+ % \bibitem[Jones et al.(1990)Jones, Baker, and Smith]{key}...
+ % The essential feature is that the label (the part in brackets) consists
+ % of the author names, as they should appear in the citation, with the year
+ % in parentheses following. There must be no space before the opening
+ % parenthesis!
+ % With natbib v5.3, a full list of authors may also follow the year.
+ % In natbib.sty, it is possible to define the type of enclosures that is
+ % really wanted (brackets or parentheses), but in either case, there must
+ % be parentheses in the label.
+ % The \cite command functions as follows:
+ % \citet{key} ==>> Jones et al. (1990)
+ % \citet*{key} ==>> Jones, Baker, and Smith (1990)
+ % \citep{key} ==>> (Jones et al., 1990)
+ % \citep*{key} ==>> (Jones, Baker, and Smith, 1990)
+ % \citep[chap. 2]{key} ==>> (Jones et al., 1990, chap. 2)
+ % \citep[e.g.][]{key} ==>> (e.g. Jones et al., 1990)
+ % \citep[e.g.][p. 32]{key} ==>> (e.g. Jones et al., 1990, p. 32)
+ % \citeauthor{key} ==>> Jones et al.
+ % \citeauthor*{key} ==>> Jones, Baker, and Smith
+ % \citeyear{key} ==>> 1990
+ %---------------------------------------------------------------------
+
+ENTRY
+ { address
+ archivePrefix
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ eid
+ eprint
+ howpublished
+ institution
+ isbn
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label short.list }
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+STRINGS { s t}
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+FUNCTION {add.blank}
+{ " " * before.all 'output.state :=
+}
+
+FUNCTION {date.block}
+{
+ new.block
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "\emph{" swap$ * "}" * }
+ if$
+}
+FUNCTION {tie.or.space.prefix}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$
+}
+
+FUNCTION {capitalize}
+{ "u" change.case$ "t" change.case$ }
+
+FUNCTION {space.word}
+{ " " swap$ * " " * }
+ % Here are the language-specific definitions for explicit words.
+ % Each function has a name bbl.xxx where xxx is the English word.
+ % The language selected here is ENGLISH
+FUNCTION {bbl.and}
+{ "and"}
+
+FUNCTION {bbl.etal}
+{ "et~al." }
+
+FUNCTION {bbl.editors}
+{ "eds." }
+
+FUNCTION {bbl.editor}
+{ "ed." }
+
+FUNCTION {bbl.edby}
+{ "edited by" }
+
+FUNCTION {bbl.edition}
+{ "edition" }
+
+FUNCTION {bbl.volume}
+{ "volume" }
+
+FUNCTION {bbl.of}
+{ "of" }
+
+FUNCTION {bbl.number}
+{ "number" }
+
+FUNCTION {bbl.nr}
+{ "no." }
+
+FUNCTION {bbl.in}
+{ "in" }
+
+FUNCTION {bbl.pages}
+{ "" }
+
+FUNCTION {bbl.page}
+{ "" }
+
+FUNCTION {bbl.chapter}
+{ "chapter" }
+
+FUNCTION {bbl.techrep}
+{ "Technical Report" }
+
+FUNCTION {bbl.mthesis}
+{ "Master's thesis" }
+
+FUNCTION {bbl.phdthesis}
+{ "Ph.D. thesis" }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+FUNCTION {bibinfo.check}
+{ swap$
+ duplicate$ missing$
+ {
+ pop$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ pop$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {bibinfo.warn}
+{ swap$
+ duplicate$ missing$
+ {
+ swap$ "missing " swap$ * " in " * cite$ * warning$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ "empty " swap$ * " in " * cite$ * warning$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.eprint}
+{ eprint duplicate$ empty$
+ 'skip$
+ { archivePrefix duplicate$ empty$
+ 'skip$
+ { ":" * swap$ }
+ if$
+ * "." *
+ }
+ if$
+}
+INTEGERS { nameptr namesleft numnames }
+
+
+STRINGS { bibinfo}
+
+FUNCTION {format.names}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}{, f.}{, jj}"
+ format.name$
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { "; " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ ";" *
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+FUNCTION {format.names.ed}
+{
+ format.names
+}
+FUNCTION {format.key}
+{ empty$
+ { key field.or.null }
+ { "" }
+ if$
+}
+
+FUNCTION {format.authors}
+{ author "author" format.names
+}
+FUNCTION {get.bbl.editor}
+{ editor num.names$ #1 > 'bbl.editors 'bbl.editor if$ }
+
+FUNCTION {format.editors}
+{ editor "editor" format.names duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ *
+ }
+ if$
+}
+FUNCTION {format.isbn}
+{ isbn "isbn" bibinfo.check
+ duplicate$ empty$ 'skip$
+ {
+ new.block
+ "ISBN " swap$ *
+ }
+ if$
+}
+
+FUNCTION {format.note}
+{
+ note empty$
+ { "" }
+ { note #1 #1 substring$
+ duplicate$ "{" =
+ 'skip$
+ { output.state mid.sentence =
+ { "l" }
+ { "u" }
+ if$
+ change.case$
+ }
+ if$
+ note #2 global.max$ substring$ * "note" bibinfo.check
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title
+ "title" bibinfo.check
+}
+FUNCTION {format.full.names}
+{'s :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}" format.name$
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { ", " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {author.editor.key.full}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.full.names }
+ if$
+ }
+ { author format.full.names }
+ if$
+}
+
+FUNCTION {author.key.full}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author format.full.names }
+ if$
+}
+
+FUNCTION {editor.key.full}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.full.names }
+ if$
+}
+
+FUNCTION {make.full.names}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.full
+ { type$ "proceedings" =
+ 'editor.key.full
+ 'author.key.full
+ if$
+ }
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+ "\bibitem[{" write$
+ label write$
+ ")" make.full.names duplicate$ short.list =
+ { pop$ }
+ { * }
+ if$
+ "}]{" * write$
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {n.dashify}
+{
+ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {word.in}
+{ bbl.in capitalize
+ " " * }
+
+FUNCTION {format.date}
+{ year "year" bibinfo.check duplicate$ empty$
+ {
+ "empty year in " cite$ * "; set to ????" * warning$
+ pop$ "????"
+ }
+ 'skip$
+ if$
+ extra.label *
+ before.all 'output.state :=
+ after.sentence 'output.state :=
+}
+FUNCTION {format.btitle}
+{ title "title" bibinfo.check
+ duplicate$ empty$ 'skip$
+ {
+ emphasize
+ }
+ if$
+}
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { bbl.volume volume tie.or.space.prefix
+ "volume" bibinfo.check * *
+ series "series" bibinfo.check
+ duplicate$ empty$ 'pop$
+ { swap$ bbl.of space.word * swap$
+ emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { series empty$
+ { number "number" bibinfo.check }
+ { output.state mid.sentence =
+ { bbl.number }
+ { bbl.number capitalize }
+ if$
+ number tie.or.space.prefix "number" bibinfo.check * *
+ bbl.in space.word *
+ series "series" bibinfo.check *
+ }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition duplicate$ empty$ 'skip$
+ {
+ output.state mid.sentence =
+ { "l" }
+ { "t" }
+ if$ change.case$
+ "edition" bibinfo.check
+ " " * bbl.edition *
+ }
+ if$
+}
+INTEGERS { multiresult }
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+FUNCTION {format.pages}
+{ pages duplicate$ empty$ 'skip$
+ { duplicate$ multi.page.check
+ {
+ n.dashify
+ }
+ {
+ }
+ if$
+ "pages" bibinfo.check
+ }
+ if$
+}
+FUNCTION {format.journal.pages}
+{ pages duplicate$ empty$ 'pop$
+ { swap$ duplicate$ empty$
+ { pop$ pop$ format.pages }
+ {
+ ": " *
+ swap$
+ n.dashify
+ "pages" bibinfo.check
+ *
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.journal.eid}
+{ eid "eid" bibinfo.check
+ duplicate$ empty$ 'pop$
+ { swap$ duplicate$ empty$ 'skip$
+ {
+ ": " *
+ }
+ if$
+ swap$ *
+ }
+ if$
+}
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ duplicate$ empty$ 'skip$
+ {
+ "volume" bibinfo.check
+ }
+ if$
+ number "number" bibinfo.check duplicate$ empty$ 'skip$
+ {
+ swap$ duplicate$ empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ swap$
+ "(" swap$ * ")" *
+ }
+ if$ *
+ eid empty$
+ { format.journal.pages }
+ { format.journal.eid }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { bbl.chapter }
+ { type "l" change.case$
+ "type" bibinfo.check
+ }
+ if$
+ chapter tie.or.space.prefix
+ "chapter" bibinfo.check
+ * *
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.booktitle}
+{
+ booktitle "booktitle" bibinfo.check
+ emphasize
+}
+FUNCTION {format.in.ed.booktitle}
+{ format.booktitle duplicate$ empty$ 'skip$
+ {
+ editor "editor" format.names.ed duplicate$ empty$ 'pop$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ ", " *
+ * swap$
+ * }
+ if$
+ word.in swap$ *
+ }
+ if$
+}
+FUNCTION {format.thesis.type}
+{ type duplicate$ empty$
+ 'pop$
+ { swap$ pop$
+ "t" change.case$ "type" bibinfo.check
+ }
+ if$
+}
+FUNCTION {format.tr.number}
+{ number "number" bibinfo.check
+ type duplicate$ empty$
+ { pop$ bbl.techrep }
+ 'skip$
+ if$
+ "type" bibinfo.check
+ swap$ duplicate$ empty$
+ { pop$ "t" change.case$ }
+ { tie.or.space.prefix * * }
+ if$
+}
+FUNCTION {format.article.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.book.crossref}
+{ volume duplicate$ empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ pop$ word.in
+ }
+ { bbl.volume
+ capitalize
+ swap$ tie.or.space.prefix "volume" bibinfo.check * * bbl.of space.word *
+ }
+ if$
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.incoll.inproc.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.org.or.pub}
+{ 't :=
+ ""
+ address empty$ t empty$ and
+ 'skip$
+ {
+ address "address" bibinfo.check *
+ t empty$
+ 'skip$
+ { address empty$
+ 'skip$
+ { ": " * }
+ if$
+ t *
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.publisher.address}
+{ publisher "publisher" bibinfo.warn format.org.or.pub
+}
+
+FUNCTION {format.organization.address}
+{ organization "organization" bibinfo.check format.org.or.pub
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ {
+ journal
+ "journal" bibinfo.check
+ emphasize
+ "journal" output.check
+ format.vol.num.pages output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check
+ editor format.key output
+ }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ format.publisher.address output
+ }
+ {
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ howpublished "howpublished" bibinfo.check output
+ address "address" bibinfo.check output
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check
+ editor format.key output
+ }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ crossref missing$
+ {
+ format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ format.publisher.address output
+ }
+ {
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ crossref missing$
+ { format.isbn output }
+ 'skip$
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ format.publisher.address output
+ format.edition output
+ format.isbn output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ new.sentence
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.isbn output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {conference} { inproceedings }
+FUNCTION {manual}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ organization address new.block.checkb
+ organization "organization" bibinfo.check output
+ address "address" bibinfo.check output
+ format.edition output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle
+ "title" output.check
+ new.block
+ bbl.mthesis format.thesis.type output.nonnull
+ school "school" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title output
+ new.block
+ howpublished "howpublished" bibinfo.check output
+ new.block
+ format.note output
+ format.eprint output
+ fin.entry
+}
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle
+ "title" output.check
+ new.block
+ bbl.phdthesis format.thesis.type output.nonnull
+ school "school" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ format.editors output
+ editor format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ new.sentence
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title
+ "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ format.note "note" output.check
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+READ
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+INTEGERS { len }
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+FUNCTION {format.lab.names}
+{'s :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}" format.name$
+ 't :=
+ nameptr #1 >
+ {
+ nameptr #2 =
+ numnames #3 > and
+ { "others" 't :=
+ #1 'namesleft := }
+ 'skip$
+ if$
+ namesleft #1 >
+ { ", " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.label}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.short.authors}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.label
+ 'author.key.label
+ if$
+ }
+ if$
+ 'short.list :=
+}
+
+FUNCTION {calc.label}
+{ calc.short.authors
+ short.list
+ "("
+ *
+ year duplicate$ empty$
+ short.list key field.or.null = or
+ { pop$ "" }
+ 'skip$
+ if$
+ *
+ 'label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv{ } }{ll{ }}{ f{ }}{ jj{ }}"
+ format.name$ 't :=
+ nameptr #1 >
+ {
+ " " *
+ namesleft #1 = t "others" = and
+ { "zzzzz" 't := }
+ 'skip$
+ if$
+ t sortify *
+ }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {editor.sort}
+{ editor empty$
+ { key empty$
+ { "to sort, need editor or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+FUNCTION {presort}
+{ calc.label
+ label sortify
+ " "
+ *
+ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.sort
+ 'author.sort
+ if$
+ }
+ if$
+ #1 entry.max$ substring$
+ 'sort.label :=
+ sort.label
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+SORT
+STRINGS { last.label next.extra }
+INTEGERS { last.extra.num last.extra.num.extended last.extra.num.blank number.label }
+FUNCTION {initialize.extra.label.stuff}
+{ #0 int.to.chr$ 'last.label :=
+ "" 'next.extra :=
+ #0 'last.extra.num :=
+ "a" chr.to.int$ #1 - 'last.extra.num.blank :=
+ last.extra.num.blank 'last.extra.num.extended :=
+ #0 'number.label :=
+}
+FUNCTION {forward.pass}
+{ last.label label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num "z" chr.to.int$ >
+ { "a" chr.to.int$ 'last.extra.num :=
+ last.extra.num.extended #1 + 'last.extra.num.extended :=
+ }
+ 'skip$
+ if$
+ last.extra.num.extended last.extra.num.blank >
+ { last.extra.num.extended int.to.chr$
+ last.extra.num int.to.chr$
+ * 'extra.label := }
+ { last.extra.num int.to.chr$ 'extra.label := }
+ if$
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ label 'last.label :=
+ }
+ if$
+ number.label #1 + 'number.label :=
+}
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+ extra.label
+ duplicate$ empty$
+ 'skip$
+ { "{\natexlab{" swap$ * "}}" * }
+ if$
+ 'extra.label :=
+ label extra.label * 'label :=
+}
+EXECUTE {initialize.extra.label.stuff}
+ITERATE {forward.pass}
+REVERSE {reverse.pass}
+FUNCTION {bib.sort.order}
+{ sort.label
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+ITERATE {bib.sort.order}
+SORT
+FUNCTION {begin.bib}
+{ preamble$ empty$
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+ "\begin{thebibliography}{" number.label int.to.str$ * "}" *
+ write$ newline$
+ "\providecommand{\natexlab}[1]{#1}"
+ write$ newline$
+}
+EXECUTE {begin.bib}
+EXECUTE {init.state.consts}
+ITERATE {call.type$}
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+EXECUTE {end.bib}
+%% End of customized bst file
+%%
+%% End of file `aaai22.bst'.
diff --git a/paper/aaai/aaai24.sty b/paper/aaai/aaai24.sty
new file mode 100644
index 0000000000000000000000000000000000000000..a68f6036d6c04077479ab116f206e30e95e6479f
--- /dev/null
+++ b/paper/aaai/aaai24.sty
@@ -0,0 +1,303 @@
+\NeedsTeXFormat{LaTeX2e}%
+\ProvidesPackage{aaai24}[2023/06/26 AAAI 2024 Submission format]%
+\def\year{2024}%
+\typeout{Conference Style for AAAI for LaTeX 2e -- version for submission}%
+%
+\def\copyright@on{T}
+\def\showauthors@on{T}
+\def\nocopyright{\gdef\copyright@on{}} % Copyright notice is required for camera-ready only.
+\DeclareOption{submission}{%
+ \gdef\copyright@on{}%
+ \gdef\showauthors@on{}%
+ \long\gdef\pdfinfo #1{\relax}%
+}%
+\ProcessOptions\relax%
+% WARNING: IF YOU ARE USING THIS STYLE SHEET FOR AN AAAI PUBLICATION, YOU
+% MAY NOT MODIFY IT FOR ANY REASON. MODIFICATIONS (IN YOUR SOURCE
+% OR IN THIS STYLE SHEET WILL RESULT IN REJECTION OF YOUR PAPER).
+%
+% WARNING: This style is NOT guaranteed to work. It is provided in the
+% hope that it might make the preparation of papers easier, but this style
+% file is provided "as is" without warranty of any kind, either express or
+% implied, including but not limited to the implied warranties of
+% merchantability, fitness for a particular purpose, or noninfringement.
+% You use this style file at your own risk. Standard disclaimers apply.
+% There are undoubtably bugs in this style. If you would like to submit
+% bug fixes, improvements, etc. please let us know. Please use the contact form
+% at www.aaai.org.
+%
+% Do not use this file unless you are an experienced LaTeX user.
+%
+% PHYSICAL PAGE LAYOUT
+\setlength\topmargin{-0.25in} \setlength\oddsidemargin{-0.25in}
+\setlength\textheight{9.0in} \setlength\textwidth{7.0in}
+\setlength\columnsep{0.375in} \newlength\titlebox \setlength\titlebox{2.25in}
+\setlength\headheight{0pt} \setlength\headsep{0pt}
+%\setlength\footheight{0pt} \setlength\footskip{0pt}
+\thispagestyle{empty} \pagestyle{empty}
+\flushbottom \twocolumn \sloppy
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip@sandia-2
+\def\addcontentsline#1#2#3{}
+% gf: PRINT COPYRIGHT NOTICE
+\def\copyright@year{\number\year}
+\def\copyright@text{Copyright \copyright\space \copyright@year,
+Association for the Advancement of Artificial Intelligence (www.aaai.org).
+All rights reserved.}
+\def\copyrighttext#1{\gdef\copyright@on{T}\gdef\copyright@text{#1}}
+\def\copyrightyear#1{\gdef\copyright@on{T}\gdef\copyright@year{#1}}
+% gf: End changes for copyright notice (used in \maketitle, below)
+% Title stuff, taken from deproc.
+%
+\def\maketitle{%
+ \par%
+ \begingroup % to make the footnote style local to the title
+ \def\thefootnote{\fnsymbol{footnote}}
+ \twocolumn[\@maketitle] \@thanks%
+ \endgroup%
+ % Insert copyright slug unless turned off
+ \if T\copyright@on\insert\footins{\noindent\footnotesize\copyright@text}\fi%
+ %
+ \setcounter{footnote}{0}%
+ \let\maketitle\relax%
+ \let\@maketitle\relax%
+ \gdef\@thanks{}%
+ \gdef\@author{}%
+ \gdef\@title{}%
+ \let\thanks\relax%
+}%
+\long\gdef\affiliations #1{ \def \affiliations_{\if T\showauthors@on#1\fi}}%
+%
+\def\@maketitle{%
+ \def\theauthors{\if T\showauthors@on\@author\else Anonymous submission\fi}
+ \newcounter{eqfn}\setcounter{eqfn}{0}%
+ \newsavebox{\titlearea}
+ \sbox{\titlearea}{
+ \let\footnote\relax\let\thanks\relax%
+ \setcounter{footnote}{0}%
+ \def\equalcontrib{%
+ \ifnum\value{eqfn}=0%
+ \footnote{These authors contributed equally.}%
+ \setcounter{eqfn}{\value{footnote}}%
+ \else%
+ \footnotemark[\value{eqfn}]%
+ \fi%
+ }%
+ \vbox{%
+ \hsize\textwidth%
+ \linewidth\hsize%
+ \vskip 0.625in minus 0.125in%
+ \centering%
+ {\LARGE\bf \@title \par}%
+ \vskip 0.1in plus 0.5fil minus 0.05in%
+ {\Large{\textbf{\theauthors\ifhmode\\\fi}}}%
+ \vskip .2em plus 0.25fil%
+ {\normalsize \affiliations_\ifhmode\\\fi}%
+ \vskip .5em plus 2fil%
+ }%
+ }%
+%
+ \newlength\actualheight%
+ \settoheight{\actualheight}{\usebox{\titlearea}}%
+ \ifdim\actualheight>\titlebox%
+ \setlength{\titlebox}{\actualheight}%
+ \fi%
+%
+ \vbox to \titlebox {%
+ \let\footnote\thanks\relax%
+ \setcounter{footnote}{0}%
+ \def\equalcontrib{%
+ \ifnum\value{eqfn}=0%
+ \footnote{These authors contributed equally.}%
+ \setcounter{eqfn}{\value{footnote}}%
+ \else%
+ \footnotemark[\value{eqfn}]%
+ \fi%
+ }%
+ \hsize\textwidth%
+ \linewidth\hsize%
+ \vskip 0.625in minus 0.125in%
+ \centering%
+ {\LARGE\bf \@title \par}%
+ \vskip 0.1in plus 0.5fil minus 0.05in%
+ {\Large{\textbf{\theauthors\ifhmode\\\fi}}}%
+ \vskip .2em plus 0.25fil%
+ {\normalsize \affiliations_\ifhmode\\\fi}%
+ \vskip .5em plus 2fil%
+ }%
+}%
+%
+\renewenvironment{abstract}{%
+ \centerline{\bf Abstract}%
+ \vspace{0.5ex}%
+ \setlength{\leftmargini}{10pt}%
+ \begin{quote}%
+ \small%
+}{%
+ \par%
+ \end{quote}%
+ \vskip 1ex%
+}%
+% jsp added:
+\def\pubnote#1{
+ \thispagestyle{myheadings}%
+ \pagestyle{myheadings}%
+ \markboth{#1}{#1}%
+ \setlength\headheight{10pt}%
+ \setlength\headsep{10pt}%
+}%
+%
+% SECTIONS with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\Large\bf\centering}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-2.0ex plus
+-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\large\bf\raggedright}}
+\def\subsubsection{\@startsection{subparagraph}{3}{\z@}{-6pt plus
+%%% DIEGO changed: 29/11/2009
+%% 2pt minus 1pt}{-1em}{\normalsize\bf}}
+-2pt minus -1pt}{-1em}{\normalsize\bf}}
+%%% END changed
+\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}{-6pt plus -2pt minus -1pt}{-1em}{\normalsize\bf}}%
+\setcounter{secnumdepth}{0}
+% add period to section (but not subsection) numbers, reduce space after
+%\renewcommand{\thesection}
+% {\arabic{section}.\hskip-0.6em}
+%\renewcommand{\thesubsection}
+% {\arabic{section}.\arabic{subsection}\hskip-0.6em}
+% FOOTNOTES
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+% LISTS AND PARAGRAPHS
+\parindent 10pt
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 0.5pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+\leftmargin 10pt \leftmargini 13pt \leftmarginii 10pt \leftmarginiii 5pt \leftmarginiv 5pt \leftmarginv 5pt \leftmarginvi 5pt
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+\labelwidth\leftmarginii\advance\labelwidth-\labelsep
+\topsep 2pt plus 1pt minus 0.5pt
+\parsep 1pt plus 0.5pt minus 0.5pt
+\itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+\labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+\topsep 1pt plus 0.5pt minus 0.5pt
+\parsep \z@
+\partopsep 0.5pt plus 0pt minus 0.5pt
+\itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+\labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+\labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+\labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\normalsize{\@setfontsize\normalsize\@xpt{11}} % 10 point on 11
+\def\small{\@setfontsize\small\@ixpt{10}} % 9 point on 10
+\def\footnotesize{\@setfontsize\footnotesize\@ixpt{10}} % 9 point on 10
+\def\scriptsize{\@setfontsize\scriptsize\@viipt{10}} % 7 point on 8
+\def\tiny{\@setfontsize\tiny\@vipt{7}} % 6 point on 7
+\def\large{\@setfontsize\large\@xipt{12}} % 11 point on 12
+\def\Large{\@setfontsize\Large\@xiipt{14}} % 12 point on 14
+\def\LARGE{\@setfontsize\LARGE\@xivpt{16}} % 14 point on 16
+\def\huge{\@setfontsize\huge\@xviipt{20}} % 17 point on 20
+\def\Huge{\@setfontsize\Huge\@xxpt{23}} % 20 point on 23
+
+\AtBeginDocument{%
+ \@ifpackageloaded{natbib}%
+ {%
+ % When natbib is in use, set the proper style and fix a few things
+ \let\cite\citep
+ \let\shortcite\citeyearpar
+ \setcitestyle{aysep={}}
+ \setlength\bibhang{0pt}
+ \bibliographystyle{aaai24}
+ }{}%
+ \@ifpackageloaded{hyperref}%
+ {%
+ \PackageError{aaai}{You must not use hyperref in AAAI papers.}{You (or one of the packages you imported) are importing the hyperref package, which is forbidden in AAAI papers. You must remove it from the paper to proceed.}
+ }{}%
+ \@ifpackageloaded{bbm}%
+ {%
+ \PackageError{aaai}{You must not use bbm package in AAAI papers because it introduces Type 3 fonts which are forbidden.}{See https://tex.stackexchange.com/questions/479160/a-replacement-to-mathbbm1-with-type-1-fonts for possible alternatives.}
+ }{}%
+ \@ifpackageloaded{authblk}%
+ {%
+ \PackageError{aaai}{Package authblk is forbbidden.}{Package authblk is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{balance}%
+ {%
+ \PackageError{aaai}{Package balance is forbbidden.}{Package balance is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{CJK}%
+ {%
+ \PackageError{aaai}{Package CJK is forbbidden.}{Package CJK is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{flushend}%
+ {%
+ \PackageError{aaai}{Package flushend is forbbidden.}{Package flushend is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{fontenc}%
+ {%
+ \PackageError{aaai}{Package fontenc is forbbidden.}{Package fontenc is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{fullpage}%
+ {%
+ \PackageError{aaai}{Package fullpage is forbbidden.}{Package fullpage is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{geometry}%
+ {%
+ \PackageError{aaai}{Package geometry is forbbidden.}{Package geometry is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{grffile}%
+ {%
+ \PackageError{aaai}{Package grffile is forbbidden.}{Package grffile is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{navigator}%
+ {%
+ \PackageError{aaai}{Package navigator is forbbidden.}{Package navigator is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{savetrees}%
+ {%
+ \PackageError{aaai}{Package savetrees is forbbidden.}{Package savetrees is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{setspace}%
+ {%
+ \PackageError{aaai}{Package setspace is forbbidden.}{Package setspace is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{stfloats}%
+ {%
+ \PackageError{aaai}{Package stfloats is forbbidden.}{Package stfloats is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{tabu}%
+ {%
+ \PackageError{aaai}{Package tabu is forbbidden.}{Package tabu is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{titlesec}%
+ {%
+ \PackageError{aaai}{Package titlesec is forbbidden.}{Package titlesec is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{tocbibind}%
+ {%
+ \PackageError{aaai}{Package tocbibind is forbbidden.}{Package tocbibind is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{ulem}%
+ {%
+ \PackageError{aaai}{Package ulem is forbbidden.}{Package ulem is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{wrapfig}%
+ {%
+ \PackageError{aaai}{Package wrapfig is forbbidden.}{Package wrapfig is forbbiden. You must find an alternative.}
+ }{}%
+}
+
+\let\endthebibliography=\endlist
diff --git a/paper/aaai/author_response.pdf b/paper/aaai/author_response.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..0c0a9b9b1b6cb67c31c97d8e3cbe167ff8dd23f3
Binary files /dev/null and b/paper/aaai/author_response.pdf differ
diff --git a/paper/aaai/author_response.qmd b/paper/aaai/author_response.qmd
new file mode 100644
index 0000000000000000000000000000000000000000..3f3b6a59afad63a5ff9008ec96b7b1e071ee1de0
--- /dev/null
+++ b/paper/aaai/author_response.qmd
@@ -0,0 +1,61 @@
+---
+title: 'Author Response'
+format:
+ pdf:
+ number-sections: true
+ number-depth: 1
+bibliography: ../bib.bib
+---
+
+Following the reviews we received for NeurIPS we have taken substantial measures to address reviewer concerns both during the initial rebuttal period and beyond.
+
+## Additional Datasets
+
+A common concern across reviewers was limited evaluation on real-world datasets. While the scope of our initial experiments was already well in line with the existing related literature, we agreed with the reviewers and have added two additional tabular datasets from the social sciences domain as well as one additional dataset from the vision domain. As mentioned in the paper, we have resorted to datasets commonly used in the literature.
+
+### A note on image datasets
+
+Related work on plausibility of counterfactuals has largely relied on small image datasets like *MNIST* [@dhurandhar2018explanations;@schut2021generating;@delaney2023counterfactual]. This may be due to the fact that generating counterfactuals for high-dimensional input data is computationally very challenging. An exception to this rule is the work on *REVISE* [@joshi2019realistic], which uses a larger image dataset. *REVISE* is suitable for this task, because it maps counterfactuals to a lower-dimensional latent space. Similarly, our proposed *ECCCo+* should also be applicable to high-dimensional input data. In our benchmarks, however, we include other generators that search directly in the input space. Since our benchmarks required us to generate a very large number of counterfactuals, it was not at this time feasible to include larger image datasets. That is despite our best efforts to optimize the code and parallelize the computations through multi-threading and multi-processing on a high-performance computing cluster.
+
+## Constraining Energy Directly
+
+In our initial work we used our unfaithfulness metric directly as a penalty term in *ECCCo*'s counterfactual search objective. This generally achieves the highest levels of faithfulness but it has several disadvantages, some of which were pointed out by the reviewers. Our new approach constrains the energy directly, which is more theoretically grounded and leads to better results across the board. Since it does not depend on generating samples through SGLD, our new approach is much more computationally efficient as well. Additionally, it also addresses the following reviewer concerns:
+
+### Results were biased with respect to unfaithfulness metric
+
+One reviewer raised concern about the fact the using the unfaithfulness metric as a penalty biases the results. This is a valid concern which we have addressed now.
+
+### Counterfactuals looked homogeneous
+
+Another reviewer pointed out that the counterfactuals generated by *ECCCo* looked homogeneous, which is also a valid concern. The observed homogeneity most likely stemmed form the fact that the samples generated through SGLD for the underlying models were fairly homogenous. With our new approach we no longer rely on SGLD samples and the homogeneity issue is no longer present.
+
+### Closeness criterium was violated
+
+A related concern was that large perturbations induced by *ECCCo* seemed to violate the closeness criterium. As we discuss in the paper, our findings do not suggest that *ECCCo* yields unnecessarily costly counterfactuals. Indeed, with reference to the vision data, *ECCCo* seems to keep useful parts of the factual largely in tact, which reduces costs. As we already argued during the rebuttal and in the paper, additional costs cannot be avoided entirely when faithfulness and plausibility are prioritized. This applies to *ECCCo* as much as to other generators like *REVISE*.
+
+### Generalizability
+
+This was not an explicit concern but some reviewers wondered if *ECCCo* could also be applied to non-differentiable models. While our initial approach that relied on SGLD samples was not suitable for non-differentiable models, our new approach is. This is because none of its penalties rely on differentiability. Of course, we still framed *ECCCo* in terms of gradient-based optimization, but the proposed penalties could be applied to other, non-gradient-based counterfactual generators as well such as *FeatureTweak*, for example [@tolomei2017interpretable].
+
+## Beyond JEMs
+
+Another common concern was that *ECCCo* primiarly achieved good results for JEMs. This has been addressed by introducing *ECCCo+* for situations when plausibility is crucial. We find that *ECCCo+* achieves good plausibility and faithfulness across the board. We have added convolutional neural networks to our analysis and find that *ECCCo+* achieves results for these models that are at least on par with the results for JEMs.
+
+## Mathematical notation and concepts
+
+One reviewer took issue with our mathematical notation, a concern that was not shared by any of the other reviewers. Nonetheless, we have revisited the notation and hope that it is now more clear. That same reviewer also raised concern about our definitions of plausibility and faithfulness that rely on distributional properties. We have extensively argued our case during the rebuttal and pointed to a potential reviwer misunderstanding in this context. None of the other reviewers found any issue with our definitions and we have made no changes in this regard. We did, however, make a minor change with respect to the related evaluation metrics. We are now more careful about our choice of the distance function. In particular, we investigated various distance metrics for image data and decided to rely on structural dissimilarity. For all other data we use the L2 Norm, where we previously used the L1 Norm. This has no impact on the results, but there was no obvious reason to use the L1 Norm in the first place other than the fact that it is typically used to assess closeness.
+
+## Conformal prediction was introduced too suddenly
+
+One reviewer pointed out that conformal prediction was introduced too suddenly. We have moved the introduction of conformal prediction forward and added more detail in line with reviewer feedback.
+
+## Limitations section
+
+We have extended the limitations section to address reviewer concerns.
+
+## Other improvements
+
+As discussed above, counterfactual explanations do not scale very well to high-dimensional input data. The NeurIPS feedback has motivated us to work on this issue by enabling intuitive support for multi-threading and multi-processing to our code. This has not only allowed us to include additional datasets but also to run extensive experiments to fine-tune hyperparameter choices. All of our code will be open-sourced as a package and we hope that it will be as useful to the community as was to us during our research.
+
+## References
+
\ No newline at end of file
diff --git a/paper/aaai/paper.pdf b/paper/aaai/paper.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..71ffb436d57ea7777615d3a59fd54c54d747d3bd
Binary files /dev/null and b/paper/aaai/paper.pdf differ
diff --git a/paper/aaai/paper.tex b/paper/aaai/paper.tex
new file mode 100644
index 0000000000000000000000000000000000000000..55ffe54ac37f0e514b590aafed2c36999ffaeabd
--- /dev/null
+++ b/paper/aaai/paper.tex
@@ -0,0 +1,198 @@
+%File: anonymous-submission-latex-2024.tex
+\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
+\usepackage[submission]{aaai24} % DO NOT CHANGE THIS
+\usepackage{times} % DO NOT CHANGE THIS
+\usepackage{helvet} % DO NOT CHANGE THIS
+\usepackage{courier} % DO NOT CHANGE THIS
+\usepackage[hyphens]{url} % DO NOT CHANGE THIS
+\usepackage{graphicx} % DO NOT CHANGE THIS
+\urlstyle{rm} % DO NOT CHANGE THIS
+\def\UrlFont{\rm} % DO NOT CHANGE THIS
+\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\frenchspacing % DO NOT CHANGE THIS
+\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
+\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
+%
+% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
+\usepackage{algorithm}
+% \usepackage{algorithmic}
+
+%
+% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
+% \usepackage{newfloat}
+% \usepackage{listings}
+% \DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
+% \lstset{%
+% basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
+% numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
+% aboveskip=0pt,belowskip=0pt,%
+% showstringspaces=false,tabsize=2,breaklines=true}
+% \floatstyle{ruled}
+% \newfloat{listing}{tb}{lst}{}
+% \floatname{listing}{Listing}
+%
+% Keep the \pdfinfo as shown here. There's no need
+% for you to add the /Title and /Author tags.
+\pdfinfo{
+/TemplateVersion (2024.1)
+}
+
+\usepackage{amsfonts} % blackboard math symbols
+\usepackage{amsmath}
+\usepackage{amsthm}
+\usepackage{caption}
+\usepackage{graphicx}
+\usepackage{algpseudocode}
+\usepackage{import}
+\usepackage{booktabs}
+\usepackage{longtable}
+\usepackage{array}
+\usepackage{multirow}
+\usepackage{placeins}
+
+
+% Numbered Environments:
+\newtheorem{definition}{Definition}[section]
+\newtheorem{question}{Research Question}[section]
+
+% Bibliography
+% \bibliographystyle{unsrtnat}
+% \setcitestyle{numbers,square,comma}
+
+% Algorithm
+\renewcommand{\algorithmicrequire}{\textbf{Input:}}
+\renewcommand{\algorithmicensure}{\textbf{Output:}}
+
+% DISALLOWED PACKAGES
+% \usepackage{authblk} -- This package is specifically forbidden
+% \usepackage{balance} -- This package is specifically forbidden
+% \usepackage{color (if used in text)
+% \usepackage{CJK} -- This package is specifically forbidden
+% \usepackage{float} -- This package is specifically forbidden
+% \usepackage{flushend} -- This package is specifically forbidden
+% \usepackage{fontenc} -- This package is specifically forbidden
+% \usepackage{fullpage} -- This package is specifically forbidden
+% \usepackage{geometry} -- This package is specifically forbidden
+% \usepackage{grffile} -- This package is specifically forbidden
+% \usepackage{hyperref} -- This package is specifically forbidden
+% \usepackage{navigator} -- This package is specifically forbidden
+% (or any other package that embeds links such as navigator or hyperref)
+% \indentfirst} -- This package is specifically forbidden
+% \layout} -- This package is specifically forbidden
+% \multicol} -- This package is specifically forbidden
+% \nameref} -- This package is specifically forbidden
+% \usepackage{savetrees} -- This package is specifically forbidden
+% \usepackage{setspace} -- This package is specifically forbidden
+% \usepackage{stfloats} -- This package is specifically forbidden
+% \usepackage{tabu} -- This package is specifically forbidden
+% \usepackage{titlesec} -- This package is specifically forbidden
+% \usepackage{tocbibind} -- This package is specifically forbidden
+% \usepackage{ulem} -- This package is specifically forbidden
+% \usepackage{wrapfig} -- This package is specifically forbidden
+% DISALLOWED COMMANDS
+% \nocopyright -- Your paper will not be published if you use this command
+% \addtolength -- This command may not be used
+% \balance -- This command may not be used
+% \baselinestretch -- Your paper will not be published if you use this command
+% \clearpage -- No page breaks of any kind may be used for the final version of your paper
+% \columnsep -- This command may not be used
+% \newpage -- No page breaks of any kind may be used for the final version of your paper
+% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
+% \pagestyle -- This command may not be used
+% \tiny -- This is not an acceptable font size.
+% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
+% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
+
+\setcounter{secnumdepth}{2} %May be changed to 1 or 2 if section numbers are desired.
+
+% The file aaai24.sty is the style file for AAAI Press
+% proceedings, working notes, and technical reports.
+%
+
+% Title
+
+% Your title must be in mixed case, not sentence case.
+% That means all verbs (including short verbs like be, is, using,and go),
+% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
+% articles, conjunctions, and prepositions are lower case unless they
+% directly follow a colon or long dash
+\title{Faithful Model Explanations through\\
+Energy-Constrained Conformal Counterfactuals}
+\author{
+ %Authors
+ % All authors must be in the same font size and format.
+ Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
+ AAAI Style Contributions by Pater Patel Schneider,
+ Sunil Issar,\\
+ J. Scott Penberthy,
+ George Ferguson,
+ Hans Guesgen,
+ Francisco Cruz\equalcontrib,
+ Marc Pujol-Gonzalez\equalcontrib
+}
+\affiliations{
+ %Afiliations
+ \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
+ % If you have multiple authors and multiple affiliations
+ % use superscripts in text and roman font to identify them.
+ % For example,
+
+ % Sunil Issar\textsuperscript{\rm 2},
+ % J. Scott Penberthy\textsuperscript{\rm 3},
+ % George Ferguson\textsuperscript{\rm 4},
+ % Hans Guesgen\textsuperscript{\rm 5}
+ % Note that the comma should be placed after the superscript
+
+ 1900 Embarcadero Road, Suite 101\\
+ Palo Alto, California 94303-3310 USA\\
+ % email address must be in roman text type, not monospace or sans serif
+ proceedings-questions@aaai.org
+%
+% See more examples next
+}
+
+%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+% \iffalse
+% \title{My Publication Title --- Single Author}
+% \author {
+% Author Name
+% }
+% \affiliations{
+% Affiliation\\
+% Affiliation Line 2\\
+% name@example.com
+% }
+% \fi
+
+% \iffalse
+% %Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+% \title{My Publication Title --- Multiple Authors}
+% \author {
+% % Authors
+% First Author Name\textsuperscript{\rm 1},
+% Second Author Name\textsuperscript{\rm 2},
+% Third Author Name\textsuperscript{\rm 1}
+% }
+% \affiliations {
+% % Affiliations
+% \textsuperscript{\rm 1}Affiliation 1\\
+% \textsuperscript{\rm 2}Affiliation 2\\
+% firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
+% }
+% \fi
+
+\begin{document}
+
+% Body of the paper
+\import{../}{body.tex}
+
+\FloatBarrier
+
+\bibliography{../bib}
+
+\onecolumn
+
+\import{../}{appendix.tex}
+
+\end{document}
diff --git a/paper/aaai/template/aaai24.bib b/paper/aaai/template/aaai24.bib
new file mode 100644
index 0000000000000000000000000000000000000000..7b7d2bcf44a7488d282c3e9b1f9079598dc99fa3
--- /dev/null
+++ b/paper/aaai/template/aaai24.bib
@@ -0,0 +1,111 @@
+@book{em:86,
+ editor = "Engelmore, Robert and Morgan, Anthony",
+ title = "Blackboard Systems",
+ year = 1986,
+ address = "Reading, Mass.",
+ publisher = "Addison-Wesley",
+}
+
+@inproceedings{c:83,
+ author = "Clancey, William J.",
+ year = 1983,
+ title = "{Communication, Simulation, and Intelligent
+Agents: Implications of Personal Intelligent Machines
+for Medical Education}",
+ booktitle="Proceedings of the Eighth International Joint Conference on Artificial Intelligence {(IJCAI-83)}",
+ pages = "556-560",
+ address = "Menlo Park, Calif",
+ publisher = "{IJCAI Organization}",
+}
+@inproceedings{c:84,
+ author = "Clancey, William J.",
+ year = 1984,
+ title = "{Classification Problem Solving}",
+ booktitle = "Proceedings of the Fourth National
+ Conference on Artificial Intelligence",
+ pages = "45-54",
+ address = "Menlo Park, Calif.",
+ publisher="AAAI Press",
+}
+@article{r:80,
+ author = {Robinson, Arthur L.},
+ title = {New Ways to Make Microcircuits Smaller},
+ volume = {208},
+ number = {4447},
+ pages = {1019--1022},
+ year = {1980},
+ doi = {10.1126/science.208.4447.1019},
+ publisher = {American Association for the Advancement of Science},
+ issn = {0036-8075},
+ URL = {https://science.sciencemag.org/content/208/4447/1019},
+ eprint = {https://science.sciencemag.org/content/208/4447/1019.full.pdf},
+ journal = {Science},
+}
+@article{r:80x,
+ author = "Robinson, Arthur L.",
+ year = 1980,
+ title = "{New Ways to Make Microcircuits Smaller---Duplicate Entry}",
+ journal = "Science",
+ volume = 208,
+ pages = "1019-1026",
+}
+@article{hcr:83,
+title = {Strategic explanations for a diagnostic consultation system},
+journal = {International Journal of Man-Machine Studies},
+volume = {20},
+number = {1},
+pages = {3-19},
+year = {1984},
+issn = {0020-7373},
+doi = {https://doi.org/10.1016/S0020-7373(84)80003-6},
+url = {https://www.sciencedirect.com/science/article/pii/S0020737384800036},
+author = {Diane Warner Hasling and William J. Clancey and Glenn Rennels},
+abstract = {This article examines the problem of automatte explanation of reasoning, especially as it relates to expert systems. By explanation we mean the ability of a program to discuss what it is doing in some understandable way. We first present a general framework in which to view explanation and review some of the research done in this area. We then focus on the explanation system for NEOMYCIN, a medical consultation program. A consultation program interactively helps a user to solve a problem. Our goal is to have NEOMYCIN explain its problem-solving strategies. An explanation of strategy describes the plan the program is using to reach a solution. Such an explanation is usually concrete, referring to aspects of the current problem situation. Abstract explanations articulate a general principle, which can be applied in different situations; such explanations are useful in teaching and in explaining by analogy. We describe the aspects of NEOMYCIN that make abstract strategic explanations possible—the representation of strategic knowledge explicitly and separately from domain knowledge— and demonstrate how this representation can be used to generate explanations.}
+}
+@article{hcrt:83,
+ author = "Hasling, Diane Warner and Clancey, William J. and Rennels, Glenn R. and Test, Thomas",
+ year = 1983,
+ title = "{Strategic Explanations in Consultation---Duplicate}",
+ journal = "The International Journal of Man-Machine Studies",
+ volume = 20,
+ number = 1,
+ pages = "3-19",
+}
+@techreport{r:86,
+ author = "Rice, James",
+ year = 1986,
+ title = "{Poligon: A System for Parallel Problem Solving}",
+ type = "Technical Report",
+ number = "KSL-86-19",
+ institution = "Dept.\ of Computer Science, Stanford Univ.",
+}
+@phdthesis{c:79,
+ author = "Clancey, William J.",
+ year = 1979,
+ title = "{Transfer of Rule-Based Expertise
+through a Tutorial Dialogue}",
+ type = "{Ph.D.} diss.",
+ school = "Dept.\ of Computer Science, Stanford Univ.",
+ address = "Stanford, Calif.",
+}
+@unpublished{c:21,
+ author = "Clancey, William J.",
+ title = "{The Engineering of Qualitative Models}",
+ year = 2021,
+ note = "Forthcoming",
+}
+@misc{c:22,
+ title={Attention Is All You Need},
+ author={Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
+ year={2017},
+ eprint={1706.03762},
+ archivePrefix={arXiv},
+ primaryClass={cs.CL}
+}
+@misc{c:23,
+ title = "Pluto: The 'Other' Red Planet",
+ author = "{NASA}",
+ howpublished = "\url{https://www.nasa.gov/nh/pluto-the-other-red-planet}",
+ year = 2015,
+ note = "Accessed: 2018-12-06"
+}
\ No newline at end of file
diff --git a/paper/aaai/template/aaai24.bst b/paper/aaai/template/aaai24.bst
new file mode 100644
index 0000000000000000000000000000000000000000..05b1d4e4414648ff46e9199f03bc66c9f77dedb5
--- /dev/null
+++ b/paper/aaai/template/aaai24.bst
@@ -0,0 +1,1493 @@
+%%
+%% This is file `aaai22.bst',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% merlin.mbs (with options: `head,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}')
+%% merlin.mbs (with options: `tail,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}')
+%% ----------------------------------------
+%% *** Natbib-compatible implementation of 'aaai' bib style ***
+%%
+ % ===============================================================
+ % IMPORTANT NOTICE:
+ % This bibliographic style (bst) file has been generated from one or
+ % more master bibliographic style (mbs) files, listed above.
+ %
+ % This generated file can be redistributed and/or modified under the terms
+ % of the LaTeX Project Public License Distributed from CTAN
+ % archives in directory macros/latex/base/lppl.txt; either
+ % version 1 of the License, or any later version.
+ % ===============================================================
+ % Name and version information of the main mbs file:
+ % \ProvidesFile{merlin.mbs}[2011/11/18 4.33 (PWD, AO, DPC)]
+ % For use with BibTeX version 0.99a or later
+ %-------------------------------------------------------------------
+ % This bibliography style file is intended for texts in ENGLISH
+ % This is an author-year citation style bibliography. As such, it is
+ % non-standard LaTeX, and requires a special package file to function properly.
+ % Such a package is natbib.sty by Patrick W. Daly
+ % The form of the \bibitem entries is
+ % \bibitem[Jones et al.(1990)]{key}...
+ % \bibitem[Jones et al.(1990)Jones, Baker, and Smith]{key}...
+ % The essential feature is that the label (the part in brackets) consists
+ % of the author names, as they should appear in the citation, with the year
+ % in parentheses following. There must be no space before the opening
+ % parenthesis!
+ % With natbib v5.3, a full list of authors may also follow the year.
+ % In natbib.sty, it is possible to define the type of enclosures that is
+ % really wanted (brackets or parentheses), but in either case, there must
+ % be parentheses in the label.
+ % The \cite command functions as follows:
+ % \citet{key} ==>> Jones et al. (1990)
+ % \citet*{key} ==>> Jones, Baker, and Smith (1990)
+ % \citep{key} ==>> (Jones et al., 1990)
+ % \citep*{key} ==>> (Jones, Baker, and Smith, 1990)
+ % \citep[chap. 2]{key} ==>> (Jones et al., 1990, chap. 2)
+ % \citep[e.g.][]{key} ==>> (e.g. Jones et al., 1990)
+ % \citep[e.g.][p. 32]{key} ==>> (e.g. Jones et al., 1990, p. 32)
+ % \citeauthor{key} ==>> Jones et al.
+ % \citeauthor*{key} ==>> Jones, Baker, and Smith
+ % \citeyear{key} ==>> 1990
+ %---------------------------------------------------------------------
+
+ENTRY
+ { address
+ archivePrefix
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ eid
+ eprint
+ howpublished
+ institution
+ isbn
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label short.list }
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+STRINGS { s t}
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+FUNCTION {add.blank}
+{ " " * before.all 'output.state :=
+}
+
+FUNCTION {date.block}
+{
+ new.block
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "\emph{" swap$ * "}" * }
+ if$
+}
+FUNCTION {tie.or.space.prefix}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$
+}
+
+FUNCTION {capitalize}
+{ "u" change.case$ "t" change.case$ }
+
+FUNCTION {space.word}
+{ " " swap$ * " " * }
+ % Here are the language-specific definitions for explicit words.
+ % Each function has a name bbl.xxx where xxx is the English word.
+ % The language selected here is ENGLISH
+FUNCTION {bbl.and}
+{ "and"}
+
+FUNCTION {bbl.etal}
+{ "et~al." }
+
+FUNCTION {bbl.editors}
+{ "eds." }
+
+FUNCTION {bbl.editor}
+{ "ed." }
+
+FUNCTION {bbl.edby}
+{ "edited by" }
+
+FUNCTION {bbl.edition}
+{ "edition" }
+
+FUNCTION {bbl.volume}
+{ "volume" }
+
+FUNCTION {bbl.of}
+{ "of" }
+
+FUNCTION {bbl.number}
+{ "number" }
+
+FUNCTION {bbl.nr}
+{ "no." }
+
+FUNCTION {bbl.in}
+{ "in" }
+
+FUNCTION {bbl.pages}
+{ "" }
+
+FUNCTION {bbl.page}
+{ "" }
+
+FUNCTION {bbl.chapter}
+{ "chapter" }
+
+FUNCTION {bbl.techrep}
+{ "Technical Report" }
+
+FUNCTION {bbl.mthesis}
+{ "Master's thesis" }
+
+FUNCTION {bbl.phdthesis}
+{ "Ph.D. thesis" }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+FUNCTION {bibinfo.check}
+{ swap$
+ duplicate$ missing$
+ {
+ pop$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ pop$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {bibinfo.warn}
+{ swap$
+ duplicate$ missing$
+ {
+ swap$ "missing " swap$ * " in " * cite$ * warning$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ "empty " swap$ * " in " * cite$ * warning$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.eprint}
+{ eprint duplicate$ empty$
+ 'skip$
+ { archivePrefix duplicate$ empty$
+ 'skip$
+ { ":" * swap$ }
+ if$
+ * "." *
+ }
+ if$
+}
+INTEGERS { nameptr namesleft numnames }
+
+
+STRINGS { bibinfo}
+
+FUNCTION {format.names}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}{, f.}{, jj}"
+ format.name$
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { "; " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ ";" *
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+FUNCTION {format.names.ed}
+{
+ format.names
+}
+FUNCTION {format.key}
+{ empty$
+ { key field.or.null }
+ { "" }
+ if$
+}
+
+FUNCTION {format.authors}
+{ author "author" format.names
+}
+FUNCTION {get.bbl.editor}
+{ editor num.names$ #1 > 'bbl.editors 'bbl.editor if$ }
+
+FUNCTION {format.editors}
+{ editor "editor" format.names duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ *
+ }
+ if$
+}
+FUNCTION {format.isbn}
+{ isbn "isbn" bibinfo.check
+ duplicate$ empty$ 'skip$
+ {
+ new.block
+ "ISBN " swap$ *
+ }
+ if$
+}
+
+FUNCTION {format.note}
+{
+ note empty$
+ { "" }
+ { note #1 #1 substring$
+ duplicate$ "{" =
+ 'skip$
+ { output.state mid.sentence =
+ { "l" }
+ { "u" }
+ if$
+ change.case$
+ }
+ if$
+ note #2 global.max$ substring$ * "note" bibinfo.check
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title
+ "title" bibinfo.check
+}
+FUNCTION {format.full.names}
+{'s :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}" format.name$
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { ", " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {author.editor.key.full}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.full.names }
+ if$
+ }
+ { author format.full.names }
+ if$
+}
+
+FUNCTION {author.key.full}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author format.full.names }
+ if$
+}
+
+FUNCTION {editor.key.full}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.full.names }
+ if$
+}
+
+FUNCTION {make.full.names}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.full
+ { type$ "proceedings" =
+ 'editor.key.full
+ 'author.key.full
+ if$
+ }
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+ "\bibitem[{" write$
+ label write$
+ ")" make.full.names duplicate$ short.list =
+ { pop$ }
+ { * }
+ if$
+ "}]{" * write$
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {n.dashify}
+{
+ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {word.in}
+{ bbl.in capitalize
+ " " * }
+
+FUNCTION {format.date}
+{ year "year" bibinfo.check duplicate$ empty$
+ {
+ "empty year in " cite$ * "; set to ????" * warning$
+ pop$ "????"
+ }
+ 'skip$
+ if$
+ extra.label *
+ before.all 'output.state :=
+ after.sentence 'output.state :=
+}
+FUNCTION {format.btitle}
+{ title "title" bibinfo.check
+ duplicate$ empty$ 'skip$
+ {
+ emphasize
+ }
+ if$
+}
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { bbl.volume volume tie.or.space.prefix
+ "volume" bibinfo.check * *
+ series "series" bibinfo.check
+ duplicate$ empty$ 'pop$
+ { swap$ bbl.of space.word * swap$
+ emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { series empty$
+ { number "number" bibinfo.check }
+ { output.state mid.sentence =
+ { bbl.number }
+ { bbl.number capitalize }
+ if$
+ number tie.or.space.prefix "number" bibinfo.check * *
+ bbl.in space.word *
+ series "series" bibinfo.check *
+ }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition duplicate$ empty$ 'skip$
+ {
+ output.state mid.sentence =
+ { "l" }
+ { "t" }
+ if$ change.case$
+ "edition" bibinfo.check
+ " " * bbl.edition *
+ }
+ if$
+}
+INTEGERS { multiresult }
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+FUNCTION {format.pages}
+{ pages duplicate$ empty$ 'skip$
+ { duplicate$ multi.page.check
+ {
+ n.dashify
+ }
+ {
+ }
+ if$
+ "pages" bibinfo.check
+ }
+ if$
+}
+FUNCTION {format.journal.pages}
+{ pages duplicate$ empty$ 'pop$
+ { swap$ duplicate$ empty$
+ { pop$ pop$ format.pages }
+ {
+ ": " *
+ swap$
+ n.dashify
+ "pages" bibinfo.check
+ *
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.journal.eid}
+{ eid "eid" bibinfo.check
+ duplicate$ empty$ 'pop$
+ { swap$ duplicate$ empty$ 'skip$
+ {
+ ": " *
+ }
+ if$
+ swap$ *
+ }
+ if$
+}
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ duplicate$ empty$ 'skip$
+ {
+ "volume" bibinfo.check
+ }
+ if$
+ number "number" bibinfo.check duplicate$ empty$ 'skip$
+ {
+ swap$ duplicate$ empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ swap$
+ "(" swap$ * ")" *
+ }
+ if$ *
+ eid empty$
+ { format.journal.pages }
+ { format.journal.eid }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { bbl.chapter }
+ { type "l" change.case$
+ "type" bibinfo.check
+ }
+ if$
+ chapter tie.or.space.prefix
+ "chapter" bibinfo.check
+ * *
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.booktitle}
+{
+ booktitle "booktitle" bibinfo.check
+ emphasize
+}
+FUNCTION {format.in.ed.booktitle}
+{ format.booktitle duplicate$ empty$ 'skip$
+ {
+ editor "editor" format.names.ed duplicate$ empty$ 'pop$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ ", " *
+ * swap$
+ * }
+ if$
+ word.in swap$ *
+ }
+ if$
+}
+FUNCTION {format.thesis.type}
+{ type duplicate$ empty$
+ 'pop$
+ { swap$ pop$
+ "t" change.case$ "type" bibinfo.check
+ }
+ if$
+}
+FUNCTION {format.tr.number}
+{ number "number" bibinfo.check
+ type duplicate$ empty$
+ { pop$ bbl.techrep }
+ 'skip$
+ if$
+ "type" bibinfo.check
+ swap$ duplicate$ empty$
+ { pop$ "t" change.case$ }
+ { tie.or.space.prefix * * }
+ if$
+}
+FUNCTION {format.article.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.book.crossref}
+{ volume duplicate$ empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ pop$ word.in
+ }
+ { bbl.volume
+ capitalize
+ swap$ tie.or.space.prefix "volume" bibinfo.check * * bbl.of space.word *
+ }
+ if$
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.incoll.inproc.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.org.or.pub}
+{ 't :=
+ ""
+ address empty$ t empty$ and
+ 'skip$
+ {
+ address "address" bibinfo.check *
+ t empty$
+ 'skip$
+ { address empty$
+ 'skip$
+ { ": " * }
+ if$
+ t *
+ }
+ if$
+ }
+ if$
+}
+FUNCTION {format.publisher.address}
+{ publisher "publisher" bibinfo.warn format.org.or.pub
+}
+
+FUNCTION {format.organization.address}
+{ organization "organization" bibinfo.check format.org.or.pub
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ {
+ journal
+ "journal" bibinfo.check
+ emphasize
+ "journal" output.check
+ format.vol.num.pages output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check
+ editor format.key output
+ }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ format.publisher.address output
+ }
+ {
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ howpublished "howpublished" bibinfo.check output
+ address "address" bibinfo.check output
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check
+ editor format.key output
+ }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ crossref missing$
+ {
+ format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ format.publisher.address output
+ }
+ {
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ crossref missing$
+ { format.isbn output }
+ 'skip$
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ format.publisher.address output
+ format.edition output
+ format.isbn output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ new.sentence
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.isbn output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {conference} { inproceedings }
+FUNCTION {manual}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ organization address new.block.checkb
+ organization "organization" bibinfo.check output
+ address "address" bibinfo.check output
+ format.edition output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle
+ "title" output.check
+ new.block
+ bbl.mthesis format.thesis.type output.nonnull
+ school "school" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title output
+ new.block
+ howpublished "howpublished" bibinfo.check output
+ new.block
+ format.note output
+ format.eprint output
+ fin.entry
+}
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle
+ "title" output.check
+ new.block
+ bbl.phdthesis format.thesis.type output.nonnull
+ school "school" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ format.editors output
+ editor format.key output
+ format.date "year" output.check
+ date.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ new.sentence
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.isbn output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title
+ "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" bibinfo.warn output
+ address "address" bibinfo.check output
+ new.block
+ format.note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+ format.date "year" output.check
+ date.block
+ format.title "title" output.check
+ new.block
+ format.note "note" output.check
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+READ
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+INTEGERS { len }
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+FUNCTION {format.lab.names}
+{'s :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}" format.name$
+ 't :=
+ nameptr #1 >
+ {
+ nameptr #2 =
+ numnames #3 > and
+ { "others" 't :=
+ #1 'namesleft := }
+ 'skip$
+ if$
+ namesleft #1 >
+ { ", " * t * }
+ {
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ {
+ numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ bbl.and
+ space.word * t *
+ }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.label}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.short.authors}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.label
+ 'author.key.label
+ if$
+ }
+ if$
+ 'short.list :=
+}
+
+FUNCTION {calc.label}
+{ calc.short.authors
+ short.list
+ "("
+ *
+ year duplicate$ empty$
+ short.list key field.or.null = or
+ { pop$ "" }
+ 'skip$
+ if$
+ *
+ 'label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv{ } }{ll{ }}{ f{ }}{ jj{ }}"
+ format.name$ 't :=
+ nameptr #1 >
+ {
+ " " *
+ namesleft #1 = t "others" = and
+ { "zzzzz" 't := }
+ 'skip$
+ if$
+ t sortify *
+ }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {editor.sort}
+{ editor empty$
+ { key empty$
+ { "to sort, need editor or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+FUNCTION {presort}
+{ calc.label
+ label sortify
+ " "
+ *
+ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.sort
+ 'author.sort
+ if$
+ }
+ if$
+ #1 entry.max$ substring$
+ 'sort.label :=
+ sort.label
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+SORT
+STRINGS { last.label next.extra }
+INTEGERS { last.extra.num last.extra.num.extended last.extra.num.blank number.label }
+FUNCTION {initialize.extra.label.stuff}
+{ #0 int.to.chr$ 'last.label :=
+ "" 'next.extra :=
+ #0 'last.extra.num :=
+ "a" chr.to.int$ #1 - 'last.extra.num.blank :=
+ last.extra.num.blank 'last.extra.num.extended :=
+ #0 'number.label :=
+}
+FUNCTION {forward.pass}
+{ last.label label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num "z" chr.to.int$ >
+ { "a" chr.to.int$ 'last.extra.num :=
+ last.extra.num.extended #1 + 'last.extra.num.extended :=
+ }
+ 'skip$
+ if$
+ last.extra.num.extended last.extra.num.blank >
+ { last.extra.num.extended int.to.chr$
+ last.extra.num int.to.chr$
+ * 'extra.label := }
+ { last.extra.num int.to.chr$ 'extra.label := }
+ if$
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ label 'last.label :=
+ }
+ if$
+ number.label #1 + 'number.label :=
+}
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+ extra.label
+ duplicate$ empty$
+ 'skip$
+ { "{\natexlab{" swap$ * "}}" * }
+ if$
+ 'extra.label :=
+ label extra.label * 'label :=
+}
+EXECUTE {initialize.extra.label.stuff}
+ITERATE {forward.pass}
+REVERSE {reverse.pass}
+FUNCTION {bib.sort.order}
+{ sort.label
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+ITERATE {bib.sort.order}
+SORT
+FUNCTION {begin.bib}
+{ preamble$ empty$
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+ "\begin{thebibliography}{" number.label int.to.str$ * "}" *
+ write$ newline$
+ "\providecommand{\natexlab}[1]{#1}"
+ write$ newline$
+}
+EXECUTE {begin.bib}
+EXECUTE {init.state.consts}
+ITERATE {call.type$}
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+EXECUTE {end.bib}
+%% End of customized bst file
+%%
+%% End of file `aaai22.bst'.
diff --git a/paper/aaai/template/aaai24.sty b/paper/aaai/template/aaai24.sty
new file mode 100644
index 0000000000000000000000000000000000000000..a68f6036d6c04077479ab116f206e30e95e6479f
--- /dev/null
+++ b/paper/aaai/template/aaai24.sty
@@ -0,0 +1,303 @@
+\NeedsTeXFormat{LaTeX2e}%
+\ProvidesPackage{aaai24}[2023/06/26 AAAI 2024 Submission format]%
+\def\year{2024}%
+\typeout{Conference Style for AAAI for LaTeX 2e -- version for submission}%
+%
+\def\copyright@on{T}
+\def\showauthors@on{T}
+\def\nocopyright{\gdef\copyright@on{}} % Copyright notice is required for camera-ready only.
+\DeclareOption{submission}{%
+ \gdef\copyright@on{}%
+ \gdef\showauthors@on{}%
+ \long\gdef\pdfinfo #1{\relax}%
+}%
+\ProcessOptions\relax%
+% WARNING: IF YOU ARE USING THIS STYLE SHEET FOR AN AAAI PUBLICATION, YOU
+% MAY NOT MODIFY IT FOR ANY REASON. MODIFICATIONS (IN YOUR SOURCE
+% OR IN THIS STYLE SHEET WILL RESULT IN REJECTION OF YOUR PAPER).
+%
+% WARNING: This style is NOT guaranteed to work. It is provided in the
+% hope that it might make the preparation of papers easier, but this style
+% file is provided "as is" without warranty of any kind, either express or
+% implied, including but not limited to the implied warranties of
+% merchantability, fitness for a particular purpose, or noninfringement.
+% You use this style file at your own risk. Standard disclaimers apply.
+% There are undoubtably bugs in this style. If you would like to submit
+% bug fixes, improvements, etc. please let us know. Please use the contact form
+% at www.aaai.org.
+%
+% Do not use this file unless you are an experienced LaTeX user.
+%
+% PHYSICAL PAGE LAYOUT
+\setlength\topmargin{-0.25in} \setlength\oddsidemargin{-0.25in}
+\setlength\textheight{9.0in} \setlength\textwidth{7.0in}
+\setlength\columnsep{0.375in} \newlength\titlebox \setlength\titlebox{2.25in}
+\setlength\headheight{0pt} \setlength\headsep{0pt}
+%\setlength\footheight{0pt} \setlength\footskip{0pt}
+\thispagestyle{empty} \pagestyle{empty}
+\flushbottom \twocolumn \sloppy
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip@sandia-2
+\def\addcontentsline#1#2#3{}
+% gf: PRINT COPYRIGHT NOTICE
+\def\copyright@year{\number\year}
+\def\copyright@text{Copyright \copyright\space \copyright@year,
+Association for the Advancement of Artificial Intelligence (www.aaai.org).
+All rights reserved.}
+\def\copyrighttext#1{\gdef\copyright@on{T}\gdef\copyright@text{#1}}
+\def\copyrightyear#1{\gdef\copyright@on{T}\gdef\copyright@year{#1}}
+% gf: End changes for copyright notice (used in \maketitle, below)
+% Title stuff, taken from deproc.
+%
+\def\maketitle{%
+ \par%
+ \begingroup % to make the footnote style local to the title
+ \def\thefootnote{\fnsymbol{footnote}}
+ \twocolumn[\@maketitle] \@thanks%
+ \endgroup%
+ % Insert copyright slug unless turned off
+ \if T\copyright@on\insert\footins{\noindent\footnotesize\copyright@text}\fi%
+ %
+ \setcounter{footnote}{0}%
+ \let\maketitle\relax%
+ \let\@maketitle\relax%
+ \gdef\@thanks{}%
+ \gdef\@author{}%
+ \gdef\@title{}%
+ \let\thanks\relax%
+}%
+\long\gdef\affiliations #1{ \def \affiliations_{\if T\showauthors@on#1\fi}}%
+%
+\def\@maketitle{%
+ \def\theauthors{\if T\showauthors@on\@author\else Anonymous submission\fi}
+ \newcounter{eqfn}\setcounter{eqfn}{0}%
+ \newsavebox{\titlearea}
+ \sbox{\titlearea}{
+ \let\footnote\relax\let\thanks\relax%
+ \setcounter{footnote}{0}%
+ \def\equalcontrib{%
+ \ifnum\value{eqfn}=0%
+ \footnote{These authors contributed equally.}%
+ \setcounter{eqfn}{\value{footnote}}%
+ \else%
+ \footnotemark[\value{eqfn}]%
+ \fi%
+ }%
+ \vbox{%
+ \hsize\textwidth%
+ \linewidth\hsize%
+ \vskip 0.625in minus 0.125in%
+ \centering%
+ {\LARGE\bf \@title \par}%
+ \vskip 0.1in plus 0.5fil minus 0.05in%
+ {\Large{\textbf{\theauthors\ifhmode\\\fi}}}%
+ \vskip .2em plus 0.25fil%
+ {\normalsize \affiliations_\ifhmode\\\fi}%
+ \vskip .5em plus 2fil%
+ }%
+ }%
+%
+ \newlength\actualheight%
+ \settoheight{\actualheight}{\usebox{\titlearea}}%
+ \ifdim\actualheight>\titlebox%
+ \setlength{\titlebox}{\actualheight}%
+ \fi%
+%
+ \vbox to \titlebox {%
+ \let\footnote\thanks\relax%
+ \setcounter{footnote}{0}%
+ \def\equalcontrib{%
+ \ifnum\value{eqfn}=0%
+ \footnote{These authors contributed equally.}%
+ \setcounter{eqfn}{\value{footnote}}%
+ \else%
+ \footnotemark[\value{eqfn}]%
+ \fi%
+ }%
+ \hsize\textwidth%
+ \linewidth\hsize%
+ \vskip 0.625in minus 0.125in%
+ \centering%
+ {\LARGE\bf \@title \par}%
+ \vskip 0.1in plus 0.5fil minus 0.05in%
+ {\Large{\textbf{\theauthors\ifhmode\\\fi}}}%
+ \vskip .2em plus 0.25fil%
+ {\normalsize \affiliations_\ifhmode\\\fi}%
+ \vskip .5em plus 2fil%
+ }%
+}%
+%
+\renewenvironment{abstract}{%
+ \centerline{\bf Abstract}%
+ \vspace{0.5ex}%
+ \setlength{\leftmargini}{10pt}%
+ \begin{quote}%
+ \small%
+}{%
+ \par%
+ \end{quote}%
+ \vskip 1ex%
+}%
+% jsp added:
+\def\pubnote#1{
+ \thispagestyle{myheadings}%
+ \pagestyle{myheadings}%
+ \markboth{#1}{#1}%
+ \setlength\headheight{10pt}%
+ \setlength\headsep{10pt}%
+}%
+%
+% SECTIONS with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\Large\bf\centering}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-2.0ex plus
+-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\large\bf\raggedright}}
+\def\subsubsection{\@startsection{subparagraph}{3}{\z@}{-6pt plus
+%%% DIEGO changed: 29/11/2009
+%% 2pt minus 1pt}{-1em}{\normalsize\bf}}
+-2pt minus -1pt}{-1em}{\normalsize\bf}}
+%%% END changed
+\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}{-6pt plus -2pt minus -1pt}{-1em}{\normalsize\bf}}%
+\setcounter{secnumdepth}{0}
+% add period to section (but not subsection) numbers, reduce space after
+%\renewcommand{\thesection}
+% {\arabic{section}.\hskip-0.6em}
+%\renewcommand{\thesubsection}
+% {\arabic{section}.\arabic{subsection}\hskip-0.6em}
+% FOOTNOTES
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+% LISTS AND PARAGRAPHS
+\parindent 10pt
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 0.5pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+\leftmargin 10pt \leftmargini 13pt \leftmarginii 10pt \leftmarginiii 5pt \leftmarginiv 5pt \leftmarginv 5pt \leftmarginvi 5pt
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+\labelwidth\leftmarginii\advance\labelwidth-\labelsep
+\topsep 2pt plus 1pt minus 0.5pt
+\parsep 1pt plus 0.5pt minus 0.5pt
+\itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+\labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+\topsep 1pt plus 0.5pt minus 0.5pt
+\parsep \z@
+\partopsep 0.5pt plus 0pt minus 0.5pt
+\itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+\labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+\labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+\labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\normalsize{\@setfontsize\normalsize\@xpt{11}} % 10 point on 11
+\def\small{\@setfontsize\small\@ixpt{10}} % 9 point on 10
+\def\footnotesize{\@setfontsize\footnotesize\@ixpt{10}} % 9 point on 10
+\def\scriptsize{\@setfontsize\scriptsize\@viipt{10}} % 7 point on 8
+\def\tiny{\@setfontsize\tiny\@vipt{7}} % 6 point on 7
+\def\large{\@setfontsize\large\@xipt{12}} % 11 point on 12
+\def\Large{\@setfontsize\Large\@xiipt{14}} % 12 point on 14
+\def\LARGE{\@setfontsize\LARGE\@xivpt{16}} % 14 point on 16
+\def\huge{\@setfontsize\huge\@xviipt{20}} % 17 point on 20
+\def\Huge{\@setfontsize\Huge\@xxpt{23}} % 20 point on 23
+
+\AtBeginDocument{%
+ \@ifpackageloaded{natbib}%
+ {%
+ % When natbib is in use, set the proper style and fix a few things
+ \let\cite\citep
+ \let\shortcite\citeyearpar
+ \setcitestyle{aysep={}}
+ \setlength\bibhang{0pt}
+ \bibliographystyle{aaai24}
+ }{}%
+ \@ifpackageloaded{hyperref}%
+ {%
+ \PackageError{aaai}{You must not use hyperref in AAAI papers.}{You (or one of the packages you imported) are importing the hyperref package, which is forbidden in AAAI papers. You must remove it from the paper to proceed.}
+ }{}%
+ \@ifpackageloaded{bbm}%
+ {%
+ \PackageError{aaai}{You must not use bbm package in AAAI papers because it introduces Type 3 fonts which are forbidden.}{See https://tex.stackexchange.com/questions/479160/a-replacement-to-mathbbm1-with-type-1-fonts for possible alternatives.}
+ }{}%
+ \@ifpackageloaded{authblk}%
+ {%
+ \PackageError{aaai}{Package authblk is forbbidden.}{Package authblk is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{balance}%
+ {%
+ \PackageError{aaai}{Package balance is forbbidden.}{Package balance is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{CJK}%
+ {%
+ \PackageError{aaai}{Package CJK is forbbidden.}{Package CJK is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{flushend}%
+ {%
+ \PackageError{aaai}{Package flushend is forbbidden.}{Package flushend is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{fontenc}%
+ {%
+ \PackageError{aaai}{Package fontenc is forbbidden.}{Package fontenc is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{fullpage}%
+ {%
+ \PackageError{aaai}{Package fullpage is forbbidden.}{Package fullpage is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{geometry}%
+ {%
+ \PackageError{aaai}{Package geometry is forbbidden.}{Package geometry is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{grffile}%
+ {%
+ \PackageError{aaai}{Package grffile is forbbidden.}{Package grffile is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{navigator}%
+ {%
+ \PackageError{aaai}{Package navigator is forbbidden.}{Package navigator is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{savetrees}%
+ {%
+ \PackageError{aaai}{Package savetrees is forbbidden.}{Package savetrees is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{setspace}%
+ {%
+ \PackageError{aaai}{Package setspace is forbbidden.}{Package setspace is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{stfloats}%
+ {%
+ \PackageError{aaai}{Package stfloats is forbbidden.}{Package stfloats is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{tabu}%
+ {%
+ \PackageError{aaai}{Package tabu is forbbidden.}{Package tabu is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{titlesec}%
+ {%
+ \PackageError{aaai}{Package titlesec is forbbidden.}{Package titlesec is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{tocbibind}%
+ {%
+ \PackageError{aaai}{Package tocbibind is forbbidden.}{Package tocbibind is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{ulem}%
+ {%
+ \PackageError{aaai}{Package ulem is forbbidden.}{Package ulem is forbbiden. You must find an alternative.}
+ }{}%
+ \@ifpackageloaded{wrapfig}%
+ {%
+ \PackageError{aaai}{Package wrapfig is forbbidden.}{Package wrapfig is forbbiden. You must find an alternative.}
+ }{}%
+}
+
+\let\endthebibliography=\endlist
diff --git a/paper/aaai/template/anonymous-submission-latex-2024.pdf b/paper/aaai/template/anonymous-submission-latex-2024.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f4b451425c344b54f76f554626d1a82d87de8c6e
Binary files /dev/null and b/paper/aaai/template/anonymous-submission-latex-2024.pdf differ
diff --git a/paper/aaai/template/anonymous-submission-latex-2024.tex b/paper/aaai/template/anonymous-submission-latex-2024.tex
new file mode 100644
index 0000000000000000000000000000000000000000..3cdc133ef6d067fb84d7e4c4123f75763bf1f104
--- /dev/null
+++ b/paper/aaai/template/anonymous-submission-latex-2024.tex
@@ -0,0 +1,796 @@
+%File: anonymous-submission-latex-2024.tex
+\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
+\usepackage[submission]{aaai24} % DO NOT CHANGE THIS
+\usepackage{times} % DO NOT CHANGE THIS
+\usepackage{helvet} % DO NOT CHANGE THIS
+\usepackage{courier} % DO NOT CHANGE THIS
+\usepackage[hyphens]{url} % DO NOT CHANGE THIS
+\usepackage{graphicx} % DO NOT CHANGE THIS
+\urlstyle{rm} % DO NOT CHANGE THIS
+\def\UrlFont{\rm} % DO NOT CHANGE THIS
+\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
+\frenchspacing % DO NOT CHANGE THIS
+\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
+\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
+%
+% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
+\usepackage{algorithm}
+\usepackage{algorithmic}
+
+%
+% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
+\usepackage{newfloat}
+\usepackage{listings}
+\DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
+\lstset{%
+ basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
+ numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
+ aboveskip=0pt,belowskip=0pt,%
+ showstringspaces=false,tabsize=2,breaklines=true}
+\floatstyle{ruled}
+\newfloat{listing}{tb}{lst}{}
+\floatname{listing}{Listing}
+%
+% Keep the \pdfinfo as shown here. There's no need
+% for you to add the /Title and /Author tags.
+\pdfinfo{
+/TemplateVersion (2024.1)
+}
+
+% DISALLOWED PACKAGES
+% \usepackage{authblk} -- This package is specifically forbidden
+% \usepackage{balance} -- This package is specifically forbidden
+% \usepackage{color (if used in text)
+% \usepackage{CJK} -- This package is specifically forbidden
+% \usepackage{float} -- This package is specifically forbidden
+% \usepackage{flushend} -- This package is specifically forbidden
+% \usepackage{fontenc} -- This package is specifically forbidden
+% \usepackage{fullpage} -- This package is specifically forbidden
+% \usepackage{geometry} -- This package is specifically forbidden
+% \usepackage{grffile} -- This package is specifically forbidden
+% \usepackage{hyperref} -- This package is specifically forbidden
+% \usepackage{navigator} -- This package is specifically forbidden
+% (or any other package that embeds links such as navigator or hyperref)
+% \indentfirst} -- This package is specifically forbidden
+% \layout} -- This package is specifically forbidden
+% \multicol} -- This package is specifically forbidden
+% \nameref} -- This package is specifically forbidden
+% \usepackage{savetrees} -- This package is specifically forbidden
+% \usepackage{setspace} -- This package is specifically forbidden
+% \usepackage{stfloats} -- This package is specifically forbidden
+% \usepackage{tabu} -- This package is specifically forbidden
+% \usepackage{titlesec} -- This package is specifically forbidden
+% \usepackage{tocbibind} -- This package is specifically forbidden
+% \usepackage{ulem} -- This package is specifically forbidden
+% \usepackage{wrapfig} -- This package is specifically forbidden
+% DISALLOWED COMMANDS
+% \nocopyright -- Your paper will not be published if you use this command
+% \addtolength -- This command may not be used
+% \balance -- This command may not be used
+% \baselinestretch -- Your paper will not be published if you use this command
+% \clearpage -- No page breaks of any kind may be used for the final version of your paper
+% \columnsep -- This command may not be used
+% \newpage -- No page breaks of any kind may be used for the final version of your paper
+% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
+% \pagestyle -- This command may not be used
+% \tiny -- This is not an acceptable font size.
+% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
+% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
+
+\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
+
+% The file aaai24.sty is the style file for AAAI Press
+% proceedings, working notes, and technical reports.
+%
+
+% Title
+
+% Your title must be in mixed case, not sentence case.
+% That means all verbs (including short verbs like be, is, using,and go),
+% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
+% articles, conjunctions, and prepositions are lower case unless they
+% directly follow a colon or long dash
+\title{AAAI Press Anonymous Submission\\Instructions for Authors Using \LaTeX{}}
+\author{
+ %Authors
+ % All authors must be in the same font size and format.
+ Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
+ AAAI Style Contributions by Pater Patel Schneider,
+ Sunil Issar,\\
+ J. Scott Penberthy,
+ George Ferguson,
+ Hans Guesgen,
+ Francisco Cruz\equalcontrib,
+ Marc Pujol-Gonzalez\equalcontrib
+}
+\affiliations{
+ %Afiliations
+ \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
+ % If you have multiple authors and multiple affiliations
+ % use superscripts in text and roman font to identify them.
+ % For example,
+
+ % Sunil Issar\textsuperscript{\rm 2},
+ % J. Scott Penberthy\textsuperscript{\rm 3},
+ % George Ferguson\textsuperscript{\rm 4},
+ % Hans Guesgen\textsuperscript{\rm 5}
+ % Note that the comma should be placed after the superscript
+
+ 1900 Embarcadero Road, Suite 101\\
+ Palo Alto, California 94303-3310 USA\\
+ % email address must be in roman text type, not monospace or sans serif
+ proceedings-questions@aaai.org
+%
+% See more examples next
+}
+
+%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+\iffalse
+\title{My Publication Title --- Single Author}
+\author {
+ Author Name
+}
+\affiliations{
+ Affiliation\\
+ Affiliation Line 2\\
+ name@example.com
+}
+\fi
+
+\iffalse
+%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
+\title{My Publication Title --- Multiple Authors}
+\author {
+ % Authors
+ First Author Name\textsuperscript{\rm 1},
+ Second Author Name\textsuperscript{\rm 2},
+ Third Author Name\textsuperscript{\rm 1}
+}
+\affiliations {
+ % Affiliations
+ \textsuperscript{\rm 1}Affiliation 1\\
+ \textsuperscript{\rm 2}Affiliation 2\\
+ firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
+}
+\fi
+
+
+% REMOVE THIS: bibentry
+% This is only needed to show inline citations in the guidelines document. You should not need it and can safely delete it.
+\usepackage{bibentry}
+% END REMOVE bibentry
+
+\begin{document}
+
+\maketitle
+
+\begin{abstract}
+AAAI creates proceedings, working notes, and technical reports directly from electronic source furnished by the authors. To ensure that all papers in the publication have a uniform appearance, authors must adhere to the following instructions.
+\end{abstract}
+
+\section{Preparing an Anonymous Submission}
+
+This document details the formatting requirements for anonymous submissions. The requirements are the same as for camera ready papers but with a few notable differences:
+
+\begin{itemize}
+ \item Anonymous submissions must not include the author names and affiliations. Write ``Anonymous Submission'' as the ``sole author'' and leave the affiliations empty.
+ \item The PDF document's metadata should be cleared with a metadata-cleaning tool before submitting it. This is to prevent leaked information from revealing your identity.
+ \item References must be anonymized whenever the reader can infer that they are to the authors' previous work.
+ \item AAAI's copyright notice should not be included as a footer in the first page.
+ \item Only the PDF version is required at this stage. No source versions will be requested, nor any copyright transfer form.
+\end{itemize}
+
+You can achieve all of the above by enabling the \texttt{submission} option when loading the \texttt{aaai24} package:
+
+\begin{quote}\begin{scriptsize}\begin{verbatim}
+\documentclass[letterpaper]{article}
+\usepackage[submission]{aaai24}
+\end{verbatim}\end{scriptsize}\end{quote}
+
+The remainder of this document are the original camera-
+ready instructions. Any contradiction of the above points
+ought to be ignored while preparing anonymous submis-
+sions.
+
+\section{Camera-Ready Guidelines}
+
+Congratulations on having a paper selected for inclusion in an AAAI Press proceedings or technical report! This document details the requirements necessary to get your accepted paper published using PDF\LaTeX{}. If you are using Microsoft Word, instructions are provided in a different document. AAAI Press does not support any other formatting software.
+
+The instructions herein are provided as a general guide for experienced \LaTeX{} users. If you do not know how to use \LaTeX{}, please obtain assistance locally. AAAI cannot provide you with support and the accompanying style files are \textbf{not} guaranteed to work. If the results you obtain are not in accordance with the specifications you received, you must correct your source file to achieve the correct result.
+
+These instructions are generic. Consequently, they do not include specific dates, page charges, and so forth. Please consult your specific written conference instructions for details regarding your submission. Please review the entire document for specific instructions that might apply to your particular situation. All authors must comply with the following:
+
+\begin{itemize}
+\item You must use the 2024 AAAI Press \LaTeX{} style file and the aaai24.bst bibliography style files, which are located in the 2024 AAAI Author Kit (aaai24.sty, aaai24.bst).
+\item You must complete, sign, and return by the deadline the AAAI copyright form (unless directed by AAAI Press to use the AAAI Distribution License instead).
+\item You must read and format your paper source and PDF according to the formatting instructions for authors.
+\item You must submit your electronic files and abstract using our electronic submission form \textbf{on time.}
+\item You must pay any required page or formatting charges to AAAI Press so that they are received by the deadline.
+\item You must check your paper before submitting it, ensuring that it compiles without error, and complies with the guidelines found in the AAAI Author Kit.
+\end{itemize}
+
+\section{Copyright}
+All papers submitted for publication by AAAI Press must be accompanied by a valid signed copyright form. They must also contain the AAAI copyright notice at the bottom of the first page of the paper. There are no exceptions to these requirements. If you fail to provide us with a signed copyright form or disable the copyright notice, we will be unable to publish your paper. There are \textbf{no exceptions} to this policy. You will find a PDF version of the AAAI copyright form in the AAAI AuthorKit. Please see the specific instructions for your conference for submission details.
+
+\section{Formatting Requirements in Brief}
+We need source and PDF files that can be used in a variety of ways and can be output on a variety of devices. The design and appearance of the paper is strictly governed by the aaai style file (aaai24.sty).
+\textbf{You must not make any changes to the aaai style file, nor use any commands, packages, style files, or macros within your own paper that alter that design, including, but not limited to spacing, floats, margins, fonts, font size, and appearance.} AAAI imposes requirements on your source and PDF files that must be followed. Most of these requirements are based on our efforts to standardize conference manuscript properties and layout. All papers submitted to AAAI for publication will be recompiled for standardization purposes. Consequently, every paper submission must comply with the following requirements:
+
+\begin{itemize}
+\item Your .tex file must compile in PDF\LaTeX{} --- (you may not include .ps or .eps figure files.)
+\item All fonts must be embedded in the PDF file --- including your figures.
+\item Modifications to the style file, whether directly or via commands in your document may not ever be made, most especially when made in an effort to avoid extra page charges or make your paper fit in a specific number of pages.
+\item No type 3 fonts may be used (even in illustrations).
+\item You may not alter the spacing above and below captions, figures, headings, and subheadings.
+\item You may not alter the font sizes of text elements, footnotes, heading elements, captions, or title information (for references and mathematics, please see the limited exceptions provided herein).
+\item You may not alter the line spacing of text.
+\item Your title must follow Title Case capitalization rules (not sentence case).
+\item \LaTeX{} documents must use the Times or Nimbus font package (you may not use Computer Modern for the text of your paper).
+\item No \LaTeX{} 209 documents may be used or submitted.
+\item Your source must not require use of fonts for non-Roman alphabets within the text itself. If your paper includes symbols in other languages (such as, but not limited to, Arabic, Chinese, Hebrew, Japanese, Thai, Russian and other Cyrillic languages), you must restrict their use to bit-mapped figures. Fonts that require non-English language support (CID and Identity-H) must be converted to outlines or 300 dpi bitmap or removed from the document (even if they are in a graphics file embedded in the document).
+\item Two-column format in AAAI style is required for all papers.
+\item The paper size for final submission must be US letter without exception.
+\item The source file must exactly match the PDF.
+\item The document margins may not be exceeded (no overfull boxes).
+\item The number of pages and the file size must be as specified for your event.
+\item No document may be password protected.
+\item Neither the PDFs nor the source may contain any embedded links or bookmarks (no hyperref or navigator packages).
+\item Your source and PDF must not have any page numbers, footers, or headers (no pagestyle commands).
+\item Your PDF must be compatible with Acrobat 5 or higher.
+\item Your \LaTeX{} source file (excluding references) must consist of a \textbf{single} file (use of the ``input" command is not allowed.
+\item Your graphics must be sized appropriately outside of \LaTeX{} (do not use the ``clip" or ``trim'' command) .
+\end{itemize}
+
+If you do not follow these requirements, your paper will be returned to you to correct the deficiencies.
+
+\section{What Files to Submit}
+You must submit the following items to ensure that your paper is published:
+\begin{itemize}
+\item A fully-compliant PDF file.
+\item Your \LaTeX{} source file submitted as a \textbf{single} .tex file (do not use the ``input" command to include sections of your paper --- every section must be in the single source file). (The only allowable exception is .bib file, which should be included separately).
+\item The bibliography (.bib) file(s).
+\item Your source must compile on our system, which includes only standard \LaTeX{} 2020 TeXLive support files.
+\item Only the graphics files used in compiling paper.
+\item The \LaTeX{}-generated files (e.g. .aux, .bbl file, PDF, etc.).
+\end{itemize}
+
+Your \LaTeX{} source will be reviewed and recompiled on our system (if it does not compile, your paper will be returned to you. \textbf{Do not submit your source in multiple text files.} Your single \LaTeX{} source file must include all your text, your bibliography (formatted using aaai24.bst), and any custom macros.
+
+Your files should work without any supporting files (other than the program itself) on any computer with a standard \LaTeX{} distribution.
+
+\textbf{Do not send files that are not actually used in the paper.} Avoid including any files not needed for compiling your paper, including, for example, this instructions file, unused graphics files, style files, additional material sent for the purpose of the paper review, intermediate build files and so forth.
+
+\textbf{Obsolete style files.} The commands for some common packages (such as some used for algorithms), may have changed. Please be certain that you are not compiling your paper using old or obsolete style files.
+
+\textbf{Final Archive.} Place your source files in a single archive which should be compressed using .zip. The final file size may not exceed 10 MB.
+Name your source file with the last (family) name of the first author, even if that is not you.
+
+
+\section{Using \LaTeX{} to Format Your Paper}
+
+The latest version of the AAAI style file is available on AAAI's website. Download this file and place it in the \TeX\ search path. Placing it in the same directory as the paper should also work. You must download the latest version of the complete AAAI Author Kit so that you will have the latest instruction set and style file.
+
+\subsection{Document Preamble}
+
+In the \LaTeX{} source for your paper, you \textbf{must} place the following lines as shown in the example in this subsection. This command set-up is for three authors. Add or subtract author and address lines as necessary, and uncomment the portions that apply to you. In most instances, this is all you need to do to format your paper in the Times font. The helvet package will cause Helvetica to be used for sans serif. These files are part of the PSNFSS2e package, which is freely available from many Internet sites (and is often part of a standard installation).
+
+Leave the setcounter for section number depth commented out and set at 0 unless you want to add section numbers to your paper. If you do add section numbers, you must uncomment this line and change the number to 1 (for section numbers), or 2 (for section and subsection numbers). The style file will not work properly with numbering of subsubsections, so do not use a number higher than 2.
+
+\subsubsection{The Following Must Appear in Your Preamble}
+\begin{quote}
+\begin{scriptsize}\begin{verbatim}
+\documentclass[letterpaper]{article}
+% DO NOT CHANGE THIS
+\usepackage[submission]{aaai24} % DO NOT CHANGE THIS
+\usepackage{times} % DO NOT CHANGE THIS
+\usepackage{helvet} % DO NOT CHANGE THIS
+\usepackage{courier} % DO NOT CHANGE THIS
+\usepackage[hyphens]{url} % DO NOT CHANGE THIS
+\usepackage{graphicx} % DO NOT CHANGE THIS
+\urlstyle{rm} % DO NOT CHANGE THIS
+\def\UrlFont{\rm} % DO NOT CHANGE THIS
+\usepackage{graphicx} % DO NOT CHANGE THIS
+\usepackage{natbib} % DO NOT CHANGE THIS
+\usepackage{caption} % DO NOT CHANGE THIS
+\frenchspacing % DO NOT CHANGE THIS
+\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
+\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
+%
+% Keep the \pdfinfo as shown here. There's no need
+% for you to add the /Title and /Author tags.
+\pdfinfo{
+/TemplateVersion (2024.1)
+}
+\end{verbatim}\end{scriptsize}
+\end{quote}
+
+\subsection{Preparing Your Paper}
+
+After the preamble above, you should prepare your paper as follows:
+\begin{quote}
+\begin{scriptsize}\begin{verbatim}
+\begin{document}
+\maketitle
+\begin{abstract}
+%...
+\end{abstract}\end{verbatim}\end{scriptsize}
+\end{quote}
+
+\noindent You should then continue with the body of your paper. Your paper must conclude with the references, which should be inserted as follows:
+\begin{quote}
+\begin{scriptsize}\begin{verbatim}
+% References and End of Paper
+% These lines must be placed at the end of your paper
+\bibliography{Bibliography-File}
+\end{document}
+\end{verbatim}\end{scriptsize}
+\end{quote}
+
+\begin{quote}
+\begin{scriptsize}\begin{verbatim}
+\begin{document}\\
+\maketitle\\
+...\\
+\bibliography{Bibliography-File}\\
+\end{document}\\
+\end{verbatim}\end{scriptsize}
+\end{quote}
+
+\subsection{Commands and Packages That May Not Be Used}
+\begin{table*}[t]
+\centering
+
+\begin{tabular}{l|l|l|l}
+\textbackslash abovecaption &
+\textbackslash abovedisplay &
+\textbackslash addevensidemargin &
+\textbackslash addsidemargin \\
+\textbackslash addtolength &
+\textbackslash baselinestretch &
+\textbackslash belowcaption &
+\textbackslash belowdisplay \\
+\textbackslash break &
+\textbackslash clearpage &
+\textbackslash clip &
+\textbackslash columnsep \\
+\textbackslash float &
+\textbackslash input &
+\textbackslash input &
+\textbackslash linespread \\
+\textbackslash newpage &
+\textbackslash pagebreak &
+\textbackslash renewcommand &
+\textbackslash setlength \\
+\textbackslash text height &
+\textbackslash tiny &
+\textbackslash top margin &
+\textbackslash trim \\
+\textbackslash vskip\{- &
+\textbackslash vspace\{- \\
+\end{tabular}
+%}
+\caption{Commands that must not be used}
+\label{table1}
+\end{table*}
+
+\begin{table}[t]
+\centering
+%\resizebox{.95\columnwidth}{!}{
+\begin{tabular}{l|l|l|l}
+ authblk & babel & cjk & dvips \\
+ epsf & epsfig & euler & float \\
+ fullpage & geometry & graphics & hyperref \\
+ layout & linespread & lmodern & maltepaper \\
+ navigator & pdfcomment & pgfplots & psfig \\
+ pstricks & t1enc & titlesec & tocbind \\
+ ulem
+\end{tabular}
+\caption{LaTeX style packages that must not be used.}
+\label{table2}
+\end{table}
+
+There are a number of packages, commands, scripts, and macros that are incompatable with aaai24.sty. The common ones are listed in tables \ref{table1} and \ref{table2}. Generally, if a command, package, script, or macro alters floats, margins, fonts, sizing, linespacing, or the presentation of the references and citations, it is unacceptable. Note that negative vskip and vspace may not be used except in certain rare occurances, and may never be used around tables, figures, captions, sections, subsections, subsubsections, or references.
+
+
+\subsection{Page Breaks}
+For your final camera ready copy, you must not use any page break commands. References must flow directly after the text without breaks. Note that some conferences require references to be on a separate page during the review process. AAAI Press, however, does not require this condition for the final paper.
+
+
+\subsection{Paper Size, Margins, and Column Width}
+Papers must be formatted to print in two-column format on 8.5 x 11 inch US letter-sized paper. The margins must be exactly as follows:
+\begin{itemize}
+\item Top margin: 1.25 inches (first page), .75 inches (others)
+\item Left margin: .75 inches
+\item Right margin: .75 inches
+\item Bottom margin: 1.25 inches
+\end{itemize}
+
+
+The default paper size in most installations of \LaTeX{} is A4. However, because we require that your electronic paper be formatted in US letter size, the preamble we have provided includes commands that alter the default to US letter size. Please note that using any other package to alter page size (such as, but not limited to the Geometry package) will result in your final paper being returned to you for correction.
+
+
+\subsubsection{Column Width and Margins.}
+To ensure maximum readability, your paper must include two columns. Each column should be 3.3 inches wide (slightly more than 3.25 inches), with a .375 inch (.952 cm) gutter of white space between the two columns. The aaai24.sty file will automatically create these columns for you.
+
+\subsection{Overlength Papers}
+If your paper is too long and you resort to formatting tricks to make it fit, it is quite likely that it will be returned to you. The best way to retain readability if the paper is overlength is to cut text, figures, or tables. There are a few acceptable ways to reduce paper size that don't affect readability. First, turn on \textbackslash frenchspacing, which will reduce the space after periods. Next, move all your figures and tables to the top of the page. Consider removing less important portions of a figure. If you use \textbackslash centering instead of \textbackslash begin\{center\} in your figure environment, you can also buy some space. For mathematical environments, you may reduce fontsize {\bf but not below 6.5 point}.
+
+
+Commands that alter page layout are forbidden. These include \textbackslash columnsep, \textbackslash float, \textbackslash topmargin, \textbackslash topskip, \textbackslash textheight, \textbackslash textwidth, \textbackslash oddsidemargin, and \textbackslash evensizemargin (this list is not exhaustive). If you alter page layout, you will be required to pay the page fee. Other commands that are questionable and may cause your paper to be rejected include \textbackslash parindent, and \textbackslash parskip. Commands that alter the space between sections are forbidden. The title sec package is not allowed. Regardless of the above, if your paper is obviously ``squeezed" it is not going to to be accepted. Options for reducing the length of a paper include reducing the size of your graphics, cutting text, or paying the extra page charge (if it is offered).
+
+
+\subsection{Type Font and Size}
+Your paper must be formatted in Times Roman or Nimbus. We will not accept papers formatted using Computer Modern or Palatino or some other font as the text or heading typeface. Sans serif, when used, should be Courier. Use Symbol or Lucida or Computer Modern for \textit{mathematics only. }
+
+Do not use type 3 fonts for any portion of your paper, including graphics. Type 3 bitmapped fonts are designed for fixed resolution printers. Most print at 300 dpi even if the printer resolution is 1200 dpi or higher. They also often cause high resolution imagesetter devices to crash. Consequently, AAAI will not accept electronic files containing obsolete type 3 fonts. Files containing those fonts (even in graphics) will be rejected. (Authors using blackboard symbols must avoid packages that use type 3 fonts.)
+
+Fortunately, there are effective workarounds that will prevent your file from embedding type 3 bitmapped fonts. The easiest workaround is to use the required times, helvet, and courier packages with \LaTeX{}2e. (Note that papers formatted in this way will still use Computer Modern for the mathematics. To make the math look good, you'll either have to use Symbol or Lucida, or you will need to install type 1 Computer Modern fonts --- for more on these fonts, see the section ``Obtaining Type 1 Computer Modern.")
+
+If you are unsure if your paper contains type 3 fonts, view the PDF in Acrobat Reader. The Properties/Fonts window will display the font name, font type, and encoding properties of all the fonts in the document. If you are unsure if your graphics contain type 3 fonts (and they are PostScript or encapsulated PostScript documents), create PDF versions of them, and consult the properties window in Acrobat Reader.
+
+The default size for your type must be ten-point with twelve-point leading (line spacing). Start all pages (except the first) directly under the top margin. (See the next section for instructions on formatting the title page.) Indent ten points when beginning a new paragraph, unless the paragraph begins directly below a heading or subheading.
+
+
+\subsubsection{Obtaining Type 1 Computer Modern for \LaTeX{}.}
+
+If you use Computer Modern for the mathematics in your paper (you cannot use it for the text) you may need to download type 1 Computer fonts. They are available without charge from the American Mathematical Society:
+http://www.ams.org/tex/type1-fonts.html.
+
+\subsubsection{Nonroman Fonts.}
+If your paper includes symbols in other languages (such as, but not limited to, Arabic, Chinese, Hebrew, Japanese, Thai, Russian and other Cyrillic languages), you must restrict their use to bit-mapped figures.
+
+\subsection{Title and Authors}
+Your title must appear centered over both text columns in sixteen-point bold type (twenty-four point leading). The title must be written in Title Case according to the Chicago Manual of Style rules. The rules are a bit involved, but in general verbs (including short verbs like be, is, using, and go), nouns, adverbs, adjectives, and pronouns should be capitalized, (including both words in hyphenated terms), while articles, conjunctions, and prepositions are lower case unless they directly follow a colon or long dash. You can use the online tool \url{https://titlecaseconverter.com/} to double-check the proper capitalization (select the "Chicago" style and mark the "Show explanations" checkbox).
+
+Author's names should appear below the title of the paper, centered in twelve-point type (with fifteen point leading), along with affiliation(s) and complete address(es) (including electronic mail address if available) in nine-point roman type (the twelve point leading). You should begin the two-column format when you come to the abstract.
+
+\subsubsection{Formatting Author Information.}
+Author information has to be set according to the following specification depending if you have one or more than one affiliation. You may not use a table nor may you employ the \textbackslash authorblk.sty package. For one or several authors from the same institution, please separate them with commas and write all affiliation directly below (one affiliation per line) using the macros \textbackslash author and \textbackslash affiliations:
+
+\begin{quote}\begin{scriptsize}\begin{verbatim}
+\author{
+ Author 1, ..., Author n\\
+}
+\affiliations {
+ Address line\\
+ ... \\
+ Address line\\
+}
+\end{verbatim}\end{scriptsize}\end{quote}
+
+
+\noindent For authors from different institutions, use \textbackslash textsuperscript \{\textbackslash rm x \} to match authors and affiliations. Notice that there should not be any spaces between the author name (or comma following it) and the superscript.
+
+\begin{quote}\begin{scriptsize}\begin{verbatim}
+\author{
+ AuthorOne,\equalcontrib\textsuperscript{\rm 1,\rm 2}
+ AuthorTwo,\equalcontrib\textsuperscript{\rm 2}
+ AuthorThree,\textsuperscript{\rm 3}\\
+ AuthorFour,\textsuperscript{\rm 4}
+ AuthorFive \textsuperscript{\rm 5}}
+}
+\affiliations {
+ \textsuperscript{\rm 1}AffiliationOne,\\
+ \textsuperscript{\rm 2}AffiliationTwo,\\
+ \textsuperscript{\rm 3}AffiliationThree,\\
+ \textsuperscript{\rm 4}AffiliationFour,\\
+ \textsuperscript{\rm 5}AffiliationFive\\
+ \{email, email\}@affiliation.com,
+ email@affiliation.com,
+ email@affiliation.com,
+ email@affiliation.com
+}
+\end{verbatim}\end{scriptsize}\end{quote}
+
+You can indicate that some authors contributed equally using the \textbackslash equalcontrib command. This will add a marker after the author names and a footnote on the first page.
+
+Note that you may want to break the author list for better visualization. You can achieve this using a simple line break (\textbackslash \textbackslash).
+
+\subsection{\LaTeX{} Copyright Notice}
+The copyright notice automatically appears if you use aaai24.sty. It has been hardcoded and may not be disabled.
+
+\subsection{Credits}
+Any credits to a sponsoring agency should appear in the acknowledgments section, unless the agency requires different placement. If it is necessary to include this information on the front page, use
+\textbackslash thanks in either the \textbackslash author or \textbackslash title commands.
+For example:
+\begin{quote}
+\begin{small}
+\textbackslash title\{Very Important Results in AI\textbackslash thanks\{This work is
+ supported by everybody.\}\}
+\end{small}
+\end{quote}
+Multiple \textbackslash thanks commands can be given. Each will result in a separate footnote indication in the author or title with the corresponding text at the botton of the first column of the document. Note that the \textbackslash thanks command is fragile. You will need to use \textbackslash protect.
+
+Please do not include \textbackslash pubnote commands in your document.
+
+\subsection{Abstract}
+Follow the example commands in this document for creation of your abstract. The command \textbackslash begin\{abstract\} will automatically indent the text block. Please do not indent it further. {Do not include references in your abstract!}
+
+\subsection{Page Numbers}
+
+Do not print any page numbers on your paper. The use of \textbackslash pagestyle is forbidden.
+
+\subsection{Text}
+The main body of the paper must be formatted in black, ten-point Times Roman with twelve-point leading (line spacing). You may not reduce font size or the linespacing. Commands that alter font size or line spacing (including, but not limited to baselinestretch, baselineshift, linespread, and others) are expressly forbidden. In addition, you may not use color in the text.
+
+\subsection{Citations}
+Citations within the text should include the author's last name and year, for example (Newell 1980). Append lower-case letters to the year in cases of ambiguity. Multiple authors should be treated as follows: (Feigenbaum and Engelmore 1988) or (Ford, Hayes, and Glymour 1992). In the case of four or more authors, list only the first author, followed by et al. (Ford et al. 1997).
+
+\subsection{Extracts}
+Long quotations and extracts should be indented ten points from the left and right margins.
+
+\begin{quote}
+This is an example of an extract or quotation. Note the indent on both sides. Quotation marks are not necessary if you offset the text in a block like this, and properly identify and cite the quotation in the text.
+
+\end{quote}
+
+\subsection{Footnotes}
+Use footnotes judiciously, taking into account that they interrupt the reading of the text. When required, they should be consecutively numbered throughout with superscript Arabic numbers. Footnotes should appear at the bottom of the page, separated from the text by a blank line space and a thin, half-point rule.
+
+\subsection{Headings and Sections}
+When necessary, headings should be used to separate major sections of your paper. Remember, you are writing a short paper, not a lengthy book! An overabundance of headings will tend to make your paper look more like an outline than a paper. The aaai24.sty package will create headings for you. Do not alter their size nor their spacing above or below.
+
+\subsubsection{Section Numbers.}
+The use of section numbers in AAAI Press papers is optional. To use section numbers in \LaTeX{}, uncomment the setcounter line in your document preamble and change the 0 to a 1. Section numbers should not be used in short poster papers and/or extended abstracts.
+
+\subsubsection{Section Headings.}
+Sections should be arranged and headed as follows:
+\begin{enumerate}
+\item Main content sections
+\item Appendices (optional)
+\item Ethical Statement (optional, unnumbered)
+\item Acknowledgements (optional, unnumbered)
+\item References (unnumbered)
+\end{enumerate}
+
+\subsubsection{Appendices.}
+Any appendices must appear after the main content. If your main sections are numbered, appendix sections must use letters instead of arabic numerals. In \LaTeX{} you can use the \texttt{\textbackslash appendix} command to achieve this effect and then use \texttt{\textbackslash section\{Heading\}} normally for your appendix sections.
+
+\subsubsection{Ethical Statement.}
+You can write a statement about the potential ethical impact of your work, including its broad societal implications, both positive and negative. If included, such statement must be written in an unnumbered section titled \emph{Ethical Statement}.
+
+\subsubsection{Acknowledgments.}
+The acknowledgments section, if included, appears right before the references and is headed ``Acknowledgments". It must not be numbered even if other sections are (use \texttt{\textbackslash section*\{Acknowledgements\}} in \LaTeX{}). This section includes acknowledgments of help from associates and colleagues, credits to sponsoring agencies, financial support, and permission to publish. Please acknowledge other contributors, grant support, and so forth, in this section. Do not put acknowledgments in a footnote on the first page. If your grant agency requires acknowledgment of the grant on page 1, limit the footnote to the required statement, and put the remaining acknowledgments at the back. Please try to limit acknowledgments to no more than three sentences.
+
+\subsubsection{References.}
+The references section should be labeled ``References" and must appear at the very end of the paper (don't end the paper with references, and then put a figure by itself on the last page). A sample list of references is given later on in these instructions. Please use a consistent format for references. Poorly prepared or sloppy references reflect badly on the quality of your paper and your research. Please prepare complete and accurate citations.
+
+\subsection{Illustrations and Figures}
+
+\begin{figure}[t]
+\centering
+\includegraphics[width=0.9\columnwidth]{figure1} % Reduce the figure size so that it is slightly narrower than the column. Don't use precise values for figure width.This setup will avoid overfull boxes.
+\caption{Using the trim and clip commands produces fragile layers that can result in disasters (like this one from an actual paper) when the color space is corrected or the PDF combined with others for the final proceedings. Crop your figures properly in a graphics program -- not in LaTeX.}
+\label{fig1}
+\end{figure}
+
+\begin{figure*}[t]
+\centering
+\includegraphics[width=0.8\textwidth]{figure2} % Reduce the figure size so that it is slightly narrower than the column.
+\caption{Adjusting the bounding box instead of actually removing the unwanted data resulted multiple layers in this paper. It also needlessly increased the PDF size. In this case, the size of the unwanted layer doubled the paper's size, and produced the following surprising results in final production. Crop your figures properly in a graphics program. Don't just alter the bounding box.}
+\label{fig2}
+\end{figure*}
+
+% Using the \centering command instead of \begin{center} ... \end{center} will save space
+% Positioning your figure at the top of the page will save space and make the paper more readable
+% Using 0.95\columnwidth in conjunction with the
+
+
+Your paper must compile in PDF\LaTeX{}. Consequently, all your figures must be .jpg, .png, or .pdf. You may not use the .gif (the resolution is too low), .ps, or .eps file format for your figures.
+
+Figures, drawings, tables, and photographs should be placed throughout the paper on the page (or the subsequent page) where they are first discussed. Do not group them together at the end of the paper. If placed at the top of the paper, illustrations may run across both columns. Figures must not invade the top, bottom, or side margin areas. Figures must be inserted using the \textbackslash usepackage\{graphicx\}. Number figures sequentially, for example, figure 1, and so on. Do not use minipage to group figures.
+
+If you normally create your figures using pgfplots, please create the figures first, and then import them as pdfs with proper bounding boxes, as the bounding and trim boxes created by pfgplots are fragile and not valid.
+
+When you include your figures, you must crop them \textbf{outside} of \LaTeX{}. The command \textbackslash includegraphics*[clip=true, viewport 0 0 10 10]{...} might result in a PDF that looks great, but the image is \textbf{not really cropped.} The full image can reappear (and obscure whatever it is overlapping) when page numbers are applied or color space is standardized. Figures \ref{fig1}, and \ref{fig2} display some unwanted results that often occur.
+
+If your paper includes illustrations that are not compatible with PDF\TeX{} (such as .eps or .ps documents), you will need to convert them. The epstopdf package will usually work for eps files. You will need to convert your ps files to PDF in either case.
+
+\subsubsection {Figure Captions.}The illustration number and caption must appear \textit{under} the illustration. Labels and other text with the actual illustration must be at least nine-point type. However, the font and size of figure captions must be 10 point roman. Do not make them smaller, bold, or italic. (Individual words may be italicized if the context requires differentiation.)
+
+\subsection{Tables}
+
+Tables should be presented in 10 point roman type. If necessary, they may be altered to 9 point type. You may not use any commands that further reduce point size below nine points. Tables that do not fit in a single column must be placed across double columns. If your table won't fit within the margins even when spanning both columns, you must split it. Do not use minipage to group tables.
+
+\subsubsection {Table Captions.} The number and caption for your table must appear \textit{under} (not above) the table. Additionally, the font and size of table captions must be 10 point roman and must be placed beneath the figure. Do not make them smaller, bold, or italic. (Individual words may be italicized if the context requires differentiation.)
+
+
+
+\subsubsection{Low-Resolution Bitmaps.}
+You may not use low-resolution (such as 72 dpi) screen-dumps and GIF files---these files contain so few pixels that they are always blurry, and illegible when printed. If they are color, they will become an indecipherable mess when converted to black and white. This is always the case with gif files, which should never be used. The resolution of screen dumps can be increased by reducing the print size of the original file while retaining the same number of pixels. You can also enlarge files by manipulating them in software such as PhotoShop. Your figures should be 300 dpi when incorporated into your document.
+
+\subsubsection{\LaTeX{} Overflow.}
+\LaTeX{} users please beware: \LaTeX{} will sometimes put portions of the figure or table or an equation in the margin. If this happens, you need to make the figure or table span both columns. If absolutely necessary, you may reduce the figure, or reformat the equation, or reconfigure the table.{ \bf Check your log file!} You must fix any overflow into the margin (that means no overfull boxes in \LaTeX{}). \textbf{Nothing is permitted to intrude into the margin or gutter.}
+
+
+\subsubsection{Using Color.}
+Use of color is restricted to figures only. It must be WACG 2.0 compliant. (That is, the contrast ratio must be greater than 4.5:1 no matter the font size.) It must be CMYK, NOT RGB. It may never be used for any portion of the text of your paper. The archival version of your paper will be printed in black and white and grayscale. The web version must be readable by persons with disabilities. Consequently, because conversion to grayscale can cause undesirable effects (red changes to black, yellow can disappear, and so forth), we strongly suggest you avoid placing color figures in your document. If you do include color figures, you must (1) use the CMYK (not RGB) colorspace and (2) be mindful of readers who may happen to have trouble distinguishing colors. Your paper must be decipherable without using color for distinction.
+
+\subsubsection{Drawings.}
+We suggest you use computer drawing software (such as Adobe Illustrator or, (if unavoidable), the drawing tools in Microsoft Word) to create your illustrations. Do not use Microsoft Publisher. These illustrations will look best if all line widths are uniform (half- to two-point in size), and you do not create labels over shaded areas. Shading should be 133 lines per inch if possible. Use Times Roman or Helvetica for all figure call-outs. \textbf{Do not use hairline width lines} --- be sure that the stroke width of all lines is at least .5 pt. Zero point lines will print on a laser printer, but will completely disappear on the high-resolution devices used by our printers.
+
+\subsubsection{Photographs and Images.}
+Photographs and other images should be in grayscale (color photographs will not reproduce well; for example, red tones will reproduce as black, yellow may turn to white, and so forth) and set to a minimum of 300 dpi. Do not prescreen images.
+
+\subsubsection{Resizing Graphics.}
+Resize your graphics \textbf{before} you include them with LaTeX. You may \textbf{not} use trim or clip options as part of your \textbackslash includegraphics command. Resize the media box of your PDF using a graphics program instead.
+
+\subsubsection{Fonts in Your Illustrations.}
+You must embed all fonts in your graphics before including them in your LaTeX document.
+
+\subsubsection{Algorithms.}
+Algorithms and/or programs are a special kind of figures. Like all illustrations, they should appear floated to the top (preferably) or bottom of the page. However, their caption should appear in the header, left-justified and enclosed between horizontal lines, as shown in Algorithm~\ref{alg:algorithm}. The algorithm body should be terminated with another horizontal line. It is up to the authors to decide whether to show line numbers or not, how to format comments, etc.
+
+In \LaTeX{} algorithms may be typeset using the {\tt algorithm} and {\tt algorithmic} packages, but you can also use one of the many other packages for the task.
+
+\begin{algorithm}[tb]
+\caption{Example algorithm}
+\label{alg:algorithm}
+\textbf{Input}: Your algorithm's input\\
+\textbf{Parameter}: Optional list of parameters\\
+\textbf{Output}: Your algorithm's output
+\begin{algorithmic}[1] %[1] enables line numbers
+\STATE Let $t=0$.
+\WHILE{condition}
+\STATE Do some action.
+\IF {conditional}
+\STATE Perform task A.
+\ELSE
+\STATE Perform task B.
+\ENDIF
+\ENDWHILE
+\STATE \textbf{return} solution
+\end{algorithmic}
+\end{algorithm}
+
+\subsubsection{Listings.}
+Listings are much like algorithms and programs. They should also appear floated to the top (preferably) or bottom of the page. Listing captions should appear in the header, left-justified and enclosed between horizontal lines as shown in Listing~\ref{lst:listing}. Terminate the body with another horizontal line and avoid any background color. Line numbers, if included, must appear within the text column.
+
+\begin{listing}[tb]%
+\caption{Example listing {\tt quicksort.hs}}%
+\label{lst:listing}%
+\begin{lstlisting}[language=Haskell]
+quicksort :: Ord a => [a] -> [a]
+quicksort [] = []
+quicksort (p:xs) = (quicksort lesser) ++ [p] ++ (quicksort greater)
+ where
+ lesser = filter (< p) xs
+ greater = filter (>= p) xs
+\end{lstlisting}
+\end{listing}
+
+\subsection{References}
+The AAAI style includes a set of definitions for use in formatting references with BibTeX. These definitions make the bibliography style fairly close to the ones specified in the Reference Examples appendix below. To use these definitions, you also need the BibTeX style file ``aaai24.bst," available in the AAAI Author Kit on the AAAI web site. Then, at the end of your paper but before \textbackslash end{document}, you need to put the following lines:
+
+\begin{quote}
+\begin{small}
+\textbackslash bibliography\{bibfile1,bibfile2,...\}
+\end{small}
+\end{quote}
+
+Please note that the aaai24.sty class already sets the bibliographystyle for you, so you do not have to place any \textbackslash bibliographystyle command in the document yourselves. The aaai24.sty file is incompatible with the hyperref and navigator packages. If you use either, your references will be garbled and your paper will be returned to you.
+
+References may be the same size as surrounding text. However, in this section (only), you may reduce the size to \textbackslash small if your paper exceeds the allowable number of pages. Making it any smaller than 9 point with 10 point linespacing, however, is not allowed. A more precise and exact method of reducing the size of your references minimally is by means of the following command: \begin{quote}
+\textbackslash fontsize\{9.8pt\}\{10.8pt\}
+\textbackslash selectfont\end{quote}
+
+\noindent You must reduce the size equally for both font size and line spacing, and may not reduce the size beyond \{9.0pt\}\{10.0pt\}.
+
+The list of files in the \textbackslash bibliography command should be the names of your BibTeX source files (that is, the .bib files referenced in your paper).
+
+The following commands are available for your use in citing references:
+\begin{quote}
+{\em \textbackslash cite:} Cites the given reference(s) with a full citation. This appears as ``(Author Year)'' for one reference, or ``(Author Year; Author Year)'' for multiple references.\smallskip\\
+{\em \textbackslash shortcite:} Cites the given reference(s) with just the year. This appears as ``(Year)'' for one reference, or ``(Year; Year)'' for multiple references.\smallskip\\
+{\em \textbackslash citeauthor:} Cites the given reference(s) with just the author name(s) and no parentheses.\smallskip\\
+{\em \textbackslash citeyear:} Cites the given reference(s) with just the date(s) and no parentheses.
+\end{quote}
+You may also use any of the \emph{natbib} citation commands.
+
+
+\section{Proofreading Your PDF}
+Please check all the pages of your PDF file. The most commonly forgotten element is the acknowledgements --- especially the correct grant number. Authors also commonly forget to add the metadata to the source, use the wrong reference style file, or don't follow the capitalization rules or comma placement for their author-title information properly. A final common problem is text (expecially equations) that runs into the margin. You will need to fix these common errors before submitting your file.
+
+\section{Improperly Formatted Files }
+In the past, AAAI has corrected improperly formatted files submitted by the authors. Unfortunately, this has become an increasingly burdensome expense that we can no longer absorb). Consequently, if your file is improperly formatted, it will be returned to you for correction.
+
+\section{Naming Your Electronic File}
+We require that you name your \LaTeX{} source file with the last name (family name) of the first author so that it can easily be differentiated from other submissions. Complete file-naming instructions will be provided to you in the submission instructions.
+
+\section{Submitting Your Electronic Files to AAAI}
+Instructions on paper submittal will be provided to you in your acceptance letter.
+
+\section{Inquiries}
+If you have any questions about the preparation or submission of your paper as instructed in this document, please contact AAAI Press at the address given below. If you have technical questions about implementation of the aaai style file, please contact an expert at your site. We do not provide technical support for \LaTeX{} or any other software package. To avoid problems, please keep your paper simple, and do not incorporate complicated macros and style files.
+
+\begin{quote}
+\noindent AAAI Press\\
+1900 Embarcadero Road, Suite 101\\
+Palo Alto, California 94303-3310 USA\\
+\textit{Telephone:} (650) 328-3123\\
+\textit{E-mail:} See the submission instructions for your particular conference or event.
+\end{quote}
+
+\section{Additional Resources}
+\LaTeX{} is a difficult program to master. If you've used that software, and this document didn't help or some items were not explained clearly, we recommend you read Michael Shell's excellent document (testflow doc.txt V1.0a 2002/08/13) about obtaining correct PS/PDF output on \LaTeX{} systems. (It was written for another purpose, but it has general application as well). It is available at www.ctan.org in the tex-archive.
+
+\appendix
+\section{Reference Examples}
+\label{sec:reference_examples}
+
+\nobibliography*
+Formatted bibliographies should look like the following examples. You should use BibTeX to generate the references. Missing fields are unacceptable when compiling references, and usually indicate that you are using the wrong type of entry (BibTeX class).
+
+\paragraph{Book with multiple authors~\nocite{em:86}} Use the \texttt{@book} class.\\[.2em]
+\bibentry{em:86}.
+
+\paragraph{Journal and magazine articles~\nocite{r:80, hcr:83}} Use the \texttt{@article} class.\\[.2em]
+\bibentry{r:80}.\\[.2em]
+\bibentry{hcr:83}.
+
+\paragraph{Proceedings paper published by a society, press or publisher~\nocite{c:83, c:84}} Use the \texttt{@inproceedings} class. You may abbreviate the \emph{booktitle} field, but make sure that the conference edition is clear.\\[.2em]
+\bibentry{c:84}.\\[.2em]
+\bibentry{c:83}.
+
+\paragraph{University technical report~\nocite{r:86}} Use the \texttt{@techreport} class.\\[.2em]
+\bibentry{r:86}.
+
+\paragraph{Dissertation or thesis~\nocite{c:79}} Use the \texttt{@phdthesis} class.\\[.2em]
+\bibentry{c:79}.
+
+\paragraph{Forthcoming publication~\nocite{c:21}} Use the \texttt{@misc} class with a \texttt{note="Forthcoming"} annotation.
+\begin{quote}
+\begin{footnotesize}
+\begin{verbatim}
+@misc(key,
+ [...]
+ note="Forthcoming",
+)
+\end{verbatim}
+\end{footnotesize}
+\end{quote}
+\bibentry{c:21}.
+
+\paragraph{ArXiv paper~\nocite{c:22}} Fetch the BibTeX entry from the "Export Bibtex Citation" link in the arXiv website. Notice it uses the \texttt{@misc} class instead of the \texttt{@article} one, and that it includes the \texttt{eprint} and \texttt{archivePrefix} keys.
+\begin{quote}
+\begin{footnotesize}
+\begin{verbatim}
+@misc(key,
+ [...]
+ eprint="xxxx.yyyy",
+ archivePrefix="arXiv",
+)
+\end{verbatim}
+\end{footnotesize}
+\end{quote}
+\bibentry{c:22}.
+
+\paragraph{Website or online resource~\nocite{c:23}} Use the \texttt{@misc} class. Add the url in the \texttt{howpublished} field and the date of access in the \texttt{note} field:
+\begin{quote}
+\begin{footnotesize}
+\begin{verbatim}
+@misc(key,
+ [...]
+ howpublished="\url{http://...}",
+ note="Accessed: YYYY-mm-dd",
+)
+\end{verbatim}
+\end{footnotesize}
+\end{quote}
+\bibentry{c:23}.
+
+\vspace{.2em}
+For the most up to date version of the AAAI reference style, please consult the \textit{AI Magazine} Author Guidelines at \url{https://aaai.org/ojs/index.php/aimagazine/about/submissions#authorGuidelines}
+
+\section{Acknowledgments}
+AAAI is especially grateful to Peter Patel Schneider for his work in implementing the original aaai.sty file, liberally using the ideas of other style hackers, including Barbara Beeton. We also acknowledge with thanks the work of George Ferguson for his guide to using the style and BibTeX files --- which has been incorporated into this document --- and Hans Guesgen, who provided several timely modifications, as well as the many others who have, from time to time, sent in suggestions on improvements to the AAAI style. We are especially grateful to Francisco Cruz, Marc Pujol-Gonzalez, and Mico Loretan for the improvements to the Bib\TeX{} and \LaTeX{} files made in 2020.
+
+The preparation of the \LaTeX{} and Bib\TeX{} files that implement these instructions was supported by Schlumberger Palo Alto Research, AT\&T Bell Laboratories, Morgan Kaufmann Publishers, The Live Oak Press, LLC, and AAAI Press. Bibliography style changes were added by Sunil Issar. \verb+\+pubnote was added by J. Scott Penberthy. George Ferguson added support for printing the AAAI copyright slug. Additional changes to aaai24.sty and aaai24.bst have been made by Francisco Cruz and Marc Pujol-Gonzalez.
+
+\bigskip
+\noindent Thank you for reading these instructions carefully. We look forward to receiving your electronic files!
+
+\bibliography{aaai24}
+
+\end{document}
diff --git a/paper/aaai/template/figure1.pdf b/paper/aaai/template/figure1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..76b3c1f6549472b13d379ff261386c1b2fee536b
Binary files /dev/null and b/paper/aaai/template/figure1.pdf differ
diff --git a/paper/aaai/template/figure2.pdf b/paper/aaai/template/figure2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..8353d7c9395dc1b4bddc97986437ab43ba692875
Binary files /dev/null and b/paper/aaai/template/figure2.pdf differ
diff --git a/paper/appendix.tex b/paper/appendix.tex
new file mode 100644
index 0000000000000000000000000000000000000000..a2234b85f0f8246d69b82cb0b9d1d9c23bc58d5d
--- /dev/null
+++ b/paper/appendix.tex
@@ -0,0 +1,389 @@
+\appendix
+\section*{Appendices}
+\renewcommand{\thesubsection}{\Alph{subsection}}
+
+The following appendices provide additional details that are relevant to the paper. Appendices~\ref{app:jem} and~\ref{app:cp} explain any tasks related to Energy-Based Modelling and Predictive Uncertainty Quantification through Conformal Prediction, respectively. Appendix~\ref{app:eccco} provides additional technical and implementation details about our proposed generator, \textit{ECCCo}, including references to our open-sourced code base. A complete overview of our experimental setup detailing our parameter choices, training procedures and initial black-box model performance can be found in Appendix~\ref{app:setup}. Finally, Appendix~\ref{app:results} reports all of our experimental results in more detail.
+
+\subsection{Energy-Based Modelling}\label{app:jem}
+
+Since we were not able to identify any existing open-source software for Energy-Based Modelling that would be flexible enough to cater to our needs, we have developed a \texttt{Julia} package from scratch. The package has been open-sourced, but to avoid compromising the double-blind review process, we refrain from providing more information at this stage. In our development we have heavily drawn on the existing literature:~\citet{du2019implicit} describe best practices for using EBM for generative modelling;~\citet{grathwohl2020your} explain how EBM can be used to train classifiers jointly for the discriminative and generative tasks. We have used the same package for training and inference, but there are some important differences between the two cases that are worth highlighting here.
+
+\subsubsection{Training: Joint Energy Models}
+
+To train our Joint Energy Models we broadly follow the approach outlined in~\citet{grathwohl2020your}. Formally, JEMs are defined by the following joint distribution:
+
+\begin{equation}
+ \begin{aligned}
+ \log p_{\theta}(\mathbf{x},\mathbf{y}) &= \log p_{\theta}(\mathbf{y}|\mathbf{x}) + \log p_{\theta}(\mathbf{x})
+ \end{aligned}
+\end{equation}
+
+Training therefore involves a standard classification loss component $L_{\text{clf}}(\theta)=-\log p_{\theta}(\mathbf{y}|\mathbf{x})$ (e.g. cross-entropy loss) as well as a generative loss component $L_{\text{gen}}(\theta)=-\log p_{\theta}(\mathbf{x})$. Analogous to how we defined the conditional distribution over inputs in Definition~\ref{def:faithful}, $p_{\theta}(\mathbf{x})$ denotes the unconditional distribution over inputs. The model gradient of this component of the loss function can be expressed as follows:
+
+\begin{equation}\label{eq:gen-true}
+ \begin{aligned}
+ \nabla_{\theta}L_{\text{gen}}(\theta)&=-\nabla_{\theta}\log p_{\theta}(\mathbf{x})=-\left(\mathbb{E}_{p(\mathbf{x})} \left\{ \nabla_{\theta} \mathcal{E}_{\theta}(\mathbf{x}) \right\} - \mathbb{E}_{p_{\theta}(\mathbf{x})} \left\{ \nabla_{\theta} \mathcal{E}_{\theta}(\mathbf{x}) \right\} \right)
+ \end{aligned}
+\end{equation}
+
+To draw samples from $p_{\theta}(\mathbf{x})$, we rely exclusively on the conditional sampling approach described in~\citet{grathwohl2020your} for both training and inference: we first draw $\mathbf{y}\sim p(\mathbf{y})$ and then sample $\mathbf{x} \sim p_{\theta}(\mathbf{x}|\mathbf{y})$~\citep{grathwohl2020your} via Equation~\ref{eq:sgld} with energy $\mathcal{E}_{\theta}(\mathbf{x}|\mathbf{y})=\mu_{\theta}(\mathbf{x})[\mathbf{y}]$ where $\mu_{\theta}: \mathcal{X} \mapsto \mathbb{R}^K$ returns the linear predictions (logits) of our classifier $M_{\theta}$. While our package also supports unconditional sampling, we found conditional sampling to work well. It is also well aligned with CE, since in this context we are interested in conditioning on the target class.
+
+As mentioned in the body of the paper, we rely on a biased sampler involving separately specified values for the step size $\epsilon$ and the standard deviation $\sigma$ of the stochastic term involving $\mathbf{r}$. Formally, our biased sampler performs updates as follows:
+
+\begin{equation}\label{eq:biased-sgld}
+ \begin{aligned}
+ \hat{\mathbf{x}}_{j+1} &\leftarrow \hat{\mathbf{x}}_j - \frac{\phi}{2} \mathcal{E}_{\theta}(\hat{\mathbf{x}}_j|\mathbf{y}^+) + \sigma \mathbf{r}_j, && j=1,...,J
+ \end{aligned}
+\end{equation}
+
+Consistent with~\citet{grathwohl2020your}, we have specified $\phi=2$ and $\sigma=0.01$ as the default values for all of our experiments. Here we have deliberately departed slightly from the notation in Equation~\ref{eq:sgld} to emphasize that we use fixed values for $\phi$ and $\sigma$, consistent with the related literature. The number of total SGLD steps $J$ varies by dataset (Table~\ref{tab:ebmparams}). Following best practices, we initialize $\mathbf{x}_0$ randomly in 5\% of all cases and sample from a buffer in all other cases. The buffer itself is randomly initialised and gradually grows to a maximum of 10,000 samples during training as $\hat{\mathbf{x}}_{J}$ is stored in each epoch~\citep{du2019implicit,grathwohl2020your}.
+
+It is important to realise that sampling is done during each training epoch, which makes training Joint Energy Models significantly harder than conventional neural classifiers. In each epoch the generated (batch of) sample(s) $\hat{\mathbf{x}}_{J}$ is used as part of the generative loss component, which compares its energy to that of observed samples $\mathbf{x}$:
+
+\begin{equation}\label{eq:gen-loss}
+ \begin{aligned}
+ L_{\text{gen}}(\theta)&\approx\mu_{\theta}(\mathbf{x})[\mathbf{y}]-\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]
+ \end{aligned}
+\end{equation}
+
+Our full training objective can be summarized as follows,
+
+\begin{equation}\label{eq:jem-loss}
+ \begin{aligned}
+ L_{\text{JEM}}(\theta) &= L_{\text{clf}}(\theta) + L_{\text{gen}}(\theta) + \lambda L_{\text{reg}}(\theta)
+ \end{aligned}
+\end{equation}
+
+where $L_{\text{reg}}(\theta)$ is a Ridge penalty (L2 norm) that regularises energy magnitudes for both observed and generated samples~\citep{du2019implicit}. We have used varying degrees of regularization depending on the dataset ($\lambda$ in Table~\ref{tab:ebmparams}).
+
+Contrary to existing work, we have not typically used the entire minibatch of training data for the generative loss component but found that using a subset of the minibatch was often sufficient in attaining decent generative performance (Table~\ref{tab:ebmparams}). This has helped to reduce the computational burden for our models, which should make it easier for others to reproduce our findings. Figures~\ref{fig:mnist-gen} and~\ref{fig:moons-gen} show generated samples for our \textit{MNIST} and \textit{Moons} data, to provide a sense of their generative property.
+
+\import{contents/}{table_ebm_params.tex}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.75\linewidth]{../artifacts/results/images/mnist_generated_JEM Ensemble.png}
+ \caption{Conditionally generated \textit{MNIST} images for our JEM Ensemble.}\label{fig:mnist-gen}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.5\linewidth]{../artifacts/results/images/moons_generated_JEM.png}
+ \caption{Conditionally generated samples (stars) for our \textit{Moons} data using a JEM.}\label{fig:moons-gen}
+\end{figure}
+\subsubsection{Inference: Quantifying Models' Generative Property}
+
+At inference time, we assume no prior knowledge about the model's generative property. This means that we do not tab into the existing buffer of generated samples for our Joint Energy Models, but instead generate conditional samples from scratch. While we have relied on the default values $\epsilon=2$ and $\sigma=0.01$ also during inference, the number of total SGLD steps was set to $J=500$ in all cases, so significantly higher than during training. For all of our synthetic datasets and models, we generated 50 conditional samples and then formed subsets containing the $n_{E}=25$ lowest-energy samples. While in practice it would be sufficient to do this once for each model and dataset, we have chosen to perform sampling separately for each individual counterfactual in our experiments to account for stochasticity. To help reduce the computational burden for our real-world datasets we have generated only 10 conditional samples each time and used all of them in our counterfactual search. Using more samples, as we originally did, had no substantial impact on our results.
+
+\subsection{Conformal Prediction}\label{app:cp}
+
+In this Appendix~\ref{app:cp} we provide some more background on CP and explain in some more detail how we have used recent advances in Conformal Training for our purposes.
+
+\subsubsection{Background on CP}
+
+Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the data. It can be used to generate prediction intervals for regression models and prediction sets for classification models. Since the literature on CE and AR is typically concerned with classification problems, we focus on the latter. A particular variant of CP called Split Conformal Prediction (SCP) is well-suited for our purposes, because it imposes only minimal restrictions on model training.
+
+Specifically, SCP involves splitting the data $\mathcal{D}_n=\{(\mathbf{x}_i,\mathbf{y}_i)\}_{i=1,...,n}$ into a proper training set $\mathcal{D}_{\text{train}}$ and a calibration set $\mathcal{D}_{\text{cal}}$. The former is used to train the classifier in any conventional fashion. The latter is then used to compute so-called nonconformity scores: $\mathcal{S}=\{s(\mathbf{x}_i,\mathbf{y}_i)\}_{i \in \mathcal{D}_{\text{cal}}}$ where $s: (\mathcal{X},\mathcal{Y}) \mapsto \mathbb{R}$ is referred to as \textit{score function}. In the context of classification, a common choice for the score function is just $s_i=1-M_{\theta}(\mathbf{x}_i)[\mathbf{y}_i]$, that is one minus the softmax output corresponding to the observed label $\mathbf{y}_i$~\citep{angelopoulos2021gentle}.
+
+Finally, classification sets are formed as follows,
+
+\begin{equation}\label{eq:scp}
+ \begin{aligned}
+ C_{\theta}(\mathbf{x}_i;\alpha)=\{\mathbf{y}: s(\mathbf{x}_i,\mathbf{y}) \le \hat{q}\}
+ \end{aligned}
+\end{equation}
+
+where $\hat{q}$ denotes the $(1-\alpha)$-quantile of $\mathcal{S}$ and $\alpha$ is a predetermined error rate. As the size of the calibration set increases, the probability that the classification set $C(\mathbf{x}_{\text{test}})$ for a newly arrived sample $\mathbf{x}_{\text{test}}$ does not cover the true test label $\mathbf{y}_{\text{test}}$ approaches $\alpha$~\citep{angelopoulos2021gentle}.
+
+Observe from Equation~\ref{eq:scp} that Conformal Prediction works on an instance-level basis, much like CE are local. The prediction set for an individual instance $\mathbf{x}_i$ depends only on the characteristics of that sample and the specified error rate. Intuitively, the set is more likely to include multiple labels for samples that are difficult to classify, so the set size is indicative of predictive uncertainty. To see why this effect is exacerbated by small choices for $\alpha$ consider the case of $\alpha=0$, which requires that the true label is covered by the prediction set with probability equal to 1.
+
+\subsubsection{Differentiability}\label{app:cp-diff}
+
+The fact that conformal classifiers produce set-valued predictions introduces a challenge: it is not immediately obvious how to use such classifiers in the context of gradient-based counterfactual search. Put differently, it is not clear how to use prediction sets in Equation~\ref{eq:general}. Fortunately, \citet{stutz2022learning} have recently proposed a framework for Conformal Training that also hinges on differentiability. Specifically, they show how Stochastic Gradient Descent can be used to train classifiers not only for the discriminative task but also for additional objectives related to Conformal Prediction. One such objective is \textit{efficiency}: for a given target error rate $\alpha$, the efficiency of a conformal classifier improves as its average prediction set size decreases. To this end, the authors introduce a smooth set size penalty defined in Equation~\ref{eq:setsize} in the body of this paper. Formally, it is defined as $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha):=\sigma\left((s(\mathbf{x}_i,\mathbf{y})-\alpha) T^{-1}\right)$ for $\mathbf{y}\in\mathcal{Y}$, where $\sigma$ is the sigmoid function and $T$ is a hyper-parameter used for temperature scaling~\citep{stutz2022learning}.
+
+In addition to the smooth set size penalty,~\citet{stutz2022learning} also propose a configurable classification loss function, that can be used to enforce coverage. For \textit{MNIST} data, we found that using this function generally improved the visual quality of the generated counterfactuals, so we used it in our experiments involving real-world data. For the synthetic dataset, visual inspection of the counterfactuals showed that using the configurable loss function sometimes led to overshooting: counterfactuals would end up deep inside the target domain but far away from the observed samples. For this reason, we instead relied on standard cross-entropy loss for our synthetic datasets. As we have noted in the body of the paper, more experimental work is certainly needed in this context. Figure~\ref{fig:cp-diff} shows the prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a \textit{JEM} trained on our \textit{Linearly Separable} data.
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../artifacts/results/images/poc_set_size.png}
+ \caption{Prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a JEM trained on our \textit{Linearly Separable} data.}\label{fig:cp-diff}
+\end{figure}
+
+\subsection{\textit{ECCCo}}\label{app:eccco}
+
+In this section, we explain \textit{ECCCo} in some more detail, briefly discuss convergence conditions for counterfactual explanations and provide details concerning the actual implementation of our framework in \texttt{Julia}.
+
+\subsubsection{Deriving the search objective}
+
+The counterfactual search objective for \textit{ECCCo} was introduced in Equation~\ref{eq:eccco} in the body of the paper. We restate this equation here for reference:
+
+\begin{equation} \label{eq:eccco-app}
+ \begin{aligned}
+ \mathbf{Z}^\prime &= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda_{1} {\text{dist}(f(\mathbf{Z}^\prime),\mathbf{x}) } \\
+ &+ \lambda_2 \mathcal{E}_{\theta}(\mathbf{Z}^\prime,\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}) + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
+ \end{aligned}
+\end{equation}
+
+We can make the connection to energy-based modeling more explicit by restating the counterfactual search objective in terms $L_{\text{JEM}}(\theta)$, which we defined in Equation~\ref{eq:jem-loss}. In particular, consider the following counterfactual search objective,
+
+\begin{equation} \label{eq:eccco-jem}
+ \begin{aligned}
+ \mathbf{Z}^\prime &= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \{ {L_{\text{JEM}}(\theta;M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda_{1} {\text{dist}(f(\mathbf{Z}^\prime),\mathbf{x}) } + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
+ \end{aligned}
+\end{equation}
+
+where we have simply used the JEM loss function as $\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)$.
+
+Now note that aside from the additional penalties in Equation~\ref{eq:eccco-app}, the only key difference between our counterfactual search objective and the joint-energy training objective is the parameter that is being optimized. In joint-energy training we optimize the objective with respect to the network weights $\theta$. Recall that $\mathcal{E}_{\theta}(\mathbf{x}|\mathbf{y})=\mu_{\theta}(\mathbf{x})[\mathbf{y}]$. Then the partial gradient with respect to the generative loss component of $L_{\text{JEM}}(\theta)$ can be expressed as follows:
+
+\begin{equation}\label{eq:jem-grad}
+ \begin{aligned}
+ \nabla_{\theta}L_{\text{gen}}(\theta) &= \nabla_{\theta}\mu_{\theta}(\mathbf{x})[\mathbf{y}]- \nabla_{\theta}\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]
+ \end{aligned}
+\end{equation}
+
+During the counterfactual search, we take the network parameters as fixed and instead optimize with respect to the counterfactual itself\footnote{Here we omit the notion of a latent search space to make the comparison easier.},
+
+\begin{equation}\label{eq:ce-grad}
+ \begin{aligned}
+ \nabla_{\mathbf{x}}L_{\text{gen}}(\theta) &= \nabla_{\mathbf{x}}\mu_{\theta}(\mathbf{x})[\mathbf{y}^+]- \nabla_{\mathbf{x}}\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}^+]=\nabla_{\mathbf{x}}\mu_{\theta}(\mathbf{x})[\mathbf{y}^+]=\nabla_{\mathbf{x}}\mathcal{E}_{\theta}(\mathbf{x}|\mathbf{y}^+)
+ \end{aligned}
+\end{equation}
+
+where the second term is equal to zero because $\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]$ is invariant with respect to $\mathbf{x}$. Since this term has zero gradients, we can remove it from the loss function altogether. For the regularization loss component of $L_{\text{JEM}}(\theta)$ we can proceed analogously such that we can rewrite Equation~\ref{eq:eccco-jem} as follows:
+
+\begin{equation} \label{eq:eccco-jem-2}
+ \begin{aligned}
+ \mathbf{Z}^\prime =& \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+) + \mathcal{E}_{\theta}(f(\mathbf{Z}^\prime)|\mathbf{y}^+) + || \mathcal{E}_{\theta}(f(\mathbf{Z}^\prime)|\mathbf{y}^+) ||_2^2} \\ &+ \lambda_{1} {\text{dist}(f(\mathbf{Z}^\prime),\mathbf{x}) } + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
+ \end{aligned}
+\end{equation}
+
+Now we notice that Equation~\ref{eq:eccco-jem-2} is equivalent to Equation~\ref{eq:eccco-app} for $\lambda_2=1$. For the sake of simplicity, we omitted the regularization component from Equation~\ref{eq:eccco} in the main text. Intuitively, taking iterative gradient steps according to Equation~\ref{eq:ce-grad} has the effect of constraining the energy of the counterfactual until. The generative property of the underlying model implicitly enters this equation through $\theta$.
+
+\subsubsection{The \textit{ECCCo} algorithm}
+
+Algorithm~\ref{alg:eccco} describes how exactly \textit{ECCCo} works. For the sake of simplicity and without loss of generality, we limit our attention to generating a single counterfactual $\mathbf{x}^\prime=f(\mathbf{z}^\prime)$. The counterfactual state $\mathbf{z}^\prime$ is initialized at the factual $\mathbf{x}$. Other forms of initialization are also suitable but not considered here. For example, one may choose at a small random perturbation to all features~\citep{slack2021counterfactual}. Next, we calibrate the model $M_{\theta}$ through split conformal prediction. Finally, we search counterfactuals through gradient descent where $\mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ denotes our loss function defined in Equation~\ref{eq:eccco}. The search terminates once the convergence criterium is met or the maximum number of iterations $T$ has been exhausted. Note that the choice of convergence criterium has important implications on the final counterfactual which we explain below.
+
+\begin{algorithm*}[h]
+ \caption{The \textit{ECCCo} generator}\label{alg:eccco}
+ \begin{algorithmic}[1]
+ \Require $\mathbf{x}, \mathbf{y}^+, M_{\theta}, \Lambda=[\lambda_1,\lambda_2,\lambda_3], \alpha, \mathcal{D}, T$ where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^+$
+ \Ensure $\mathbf{x}^\prime$
+ \State Initialize $\mathbf{z}^\prime \gets \mathbf{x}$
+ \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$ \Comment{Calibrate model through split conformal prediction.}
+ \State Initialize $t \gets 0$
+ \While{\textit{not converged} or $t < T$} \Comment{For convergence conditions see below.}
+ \State $\mathbf{z}^\prime \gets \mathbf{z}^\prime - \eta \nabla_{\mathbf{z}^\prime} \mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+; \Lambda, \alpha)$ \Comment{Take gradient step of size $\eta$.}
+ \State $t \gets t+1$
+ \EndWhile
+ \State $\mathbf{x}^\prime \gets \mathbf{z}^\prime$
+ \end{algorithmic}
+\end{algorithm*}
+
+\subsubsection{The \textit{ECCCo+} algorithm}
+
+Algorithm~\ref{alg:eccco-plus} describes how exactly \textit{ECCCo+} works. The only difference to \textit{ECCCo} is that we encode and decode features using PCA. In particular, we let $f^{-1}(\mathbf{x})$ denote the projection of $\mathbf{x}$ to its first $n_z$ principal components. Conversely, $f(\mathbf{z}^\prime)$ maps back from the projection to the feature space.
+
+\begin{algorithm*}[h]
+ \caption{The \textit{ECCCo+} generator}\label{alg:eccco-plus}
+ \begin{algorithmic}[1]
+ \Require $\mathbf{x}, \mathbf{y}^+, M_{\theta}, f, \Lambda=[\lambda_1,\lambda_2,\lambda_3], \alpha, \mathcal{D}, T$ where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^+$
+ \Ensure $\mathbf{x}^\prime$
+ \State Initialize $\mathbf{z}^\prime \gets f^{-1}(\mathbf{x})$ \Comment{Map to counterfactual state space.}
+ \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$ \Comment{Calibrate model through split conformal prediction.}
+ \State Initialize $t \gets 0$
+ \While{\textit{not converged} or $t < T$} \Comment{For convergence conditions see below.}
+ \State $\mathbf{z}^\prime \gets \mathbf{z}^\prime - \eta \nabla_{\mathbf{z}^\prime} \mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+; \Lambda, \alpha)$ \Comment{Take gradient step of size $\eta$.}
+ \State $t \gets t+1$
+ \EndWhile
+ \State $\mathbf{x}^\prime \gets f(\mathbf{z}^\prime)$ \Comment{Map back to feature space.}
+ \end{algorithmic}
+\end{algorithm*}
+
+\subsubsection{The \textit{ECCCo-L1} algorithm}
+
+Algorithm~\ref{alg:eccco-l1} describes a variation of \textit{ECCCo} that we initally considered but ultimately discarded. For the sake of completeness we have included this approach here in the appendix. It generally yields very faithful counterfactuals but it is computationally much more expensive and struggles with plausibility.
+
+Instead of constraining energy directly, this approach works under the premise of penalizing the distance between the counterfactual and samples generated through SGLD. The counterfactual state $\mathbf{z}^\prime$ is initialized as in Algorithm~\ref{alg:eccco}. Next, we generate $n_{\mathcal{B}}$ conditional samples $\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}$ using SGLD (Equation~\ref{eq:sgld}) and store the $n_E$ instances with the lowest energy. We then calibrate the model $M_{\theta}$ through split conformal prediction. Finally, we search counterfactuals through gradient descent where $\mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ denotes our loss function defined in Equation~\ref{eq:eccco}, but instead of constraining energy directly we use Equaqtion~\ref{eq:faith} (unfaithfulness metric) as a penalty.
+
+\begin{algorithm*}[h]
+ \caption{The \textit{ECCCo-L1} generator}\label{alg:eccco-l1}
+ \begin{algorithmic}[1]
+ \Require $\mathbf{x}, \mathbf{y}^+, M_{\theta}, f, \Lambda=[\lambda_1,\lambda_2,\lambda_3], \alpha, \mathcal{D}, T, \eta, n_{\mathcal{B}}, n_E$ where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^+$
+ \Ensure $\mathbf{x}^\prime$
+ \State Initialize $\mathbf{z}^\prime \gets \mathbf{x}$
+ \State Generate $\left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}} \gets p_{\theta}(\mathbf{x}_{\mathbf{y}^+})$ \Comment{Generate $n_{\mathcal{B}}$ samples using SGLD (Equation~\ref{eq:sgld}).}
+ \State Store $\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+} \gets \left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}}$ \Comment{Choose $n_E$ lowest-energy samples.}
+ \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$ \Comment{Calibrate model through split conformal prediction.}
+ \State Initialize $t \gets 0$
+ \While{\textit{not converged} or $t < T$} \Comment{For convergence conditions see below.}
+ \State $\mathbf{z}^\prime \gets \mathbf{z}^\prime - \eta \nabla_{\mathbf{z}^\prime} \mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ \Comment{Take gradient step of size $\eta$.}
+ \State $t \gets t+1$
+ \EndWhile
+ \State $\mathbf{x}^\prime \gets \mathbf{z}^\prime$
+ \end{algorithmic}
+\end{algorithm*}
+
+\subsubsection{A Note on Convergence}\label{convergence}
+
+Convergence is not typically discussed much in the context of CE, even though it has important implications on outcomes. One intuitive way to specify convergence is in terms of threshold probabilities: once the predicted probability $p(\mathbf{y}^+|\mathbf{x}^{\prime})$ exceeds some user-defined threshold $\gamma$ such that the counterfactual is valid, we could consider the search to have converged. In the binary case, for example, convergence could be defined as $p(\mathbf{y}^+|\mathbf{x}^{\prime})>0.5$ in this sense. Note, however, how this can be expected to yield counterfactuals in the proximity of the decision boundary, a region characterized by high aleatoric uncertainty. In other words, counterfactuals generated in this way would generally not be plausible. To avoid this from happening, we specify convergence in terms of gradients approaching zero for all our experiments and all of our generators. This is allows us to get a cleaner read on how the different counterfactual search objectives affect counterfactual outcomes.
+
+\subsubsection{\texttt{ECCCo.jl}}
+
+The core part of our code base is integrated into a larger ecosystem of \texttt{Julia} packages that we are actively developing and maintaining. To avoid compromising the double-blind review process, we only provide a link to an anonymized repository at this stage: \url{https://anonymous.4open.science/r/ECCCo-1252/README.md}.
+
+\subsection{Experimental Setup}\label{app:setup}
+
+In our experiments we always generate multiple counterfactuals for each model and generator. Each time the factual and target class is drawn randomly. For each generator and model we choose $n_f=100$ factuals for all of our synthetic and vision data. For tabular data we choose $n_f=25$ because larger values made grid search computationally prohibitive. For vision data, grid search was computationally prohibitve in any case, so hyperparameters were tuned manually. For all other datasets grid search was performed over different combinations of penalty strengths and optimizer steps sizes.
+
+Table~\ref{tab:params} provides an overview of all parameters related to our experiments. The \textit{GMSC} data were randomly undersampled for balancing purposes and all features were standardized. \textit{MNIST} data was also randomly undersampled for reasons outlined below. Pixel values were preprocessed to fall in the range of $[-1,1]$ and a small Gaussian noise component ($\sigma=0.03$) was added to training samples following common practice in the EBM literature. All of our models were trained through mini-batch training using the Adam optimiser (\citet{kingma2014adam}). Table~\ref{tab:perf} shows standard evaluation metrics measuring the predictive performance of our different models grouped by dataset. These measures were computed on test data.
+
+Table~\ref{tab:genparams} summarises our hyperparameter choices for the counterfactual generators where $\eta$ denotes the learning rate used for Stochastic Gradient Descent (SGD) and $\lambda_1$, $\lambda_2$, $\lambda_3$ represent the chosen penalty strengths (Equations~\ref{eq:general} and~\ref{eq:eccco}). Here $\lambda_1$ also refers to the chosen penalty for the distance from factual values that applies to both \textit{Wachter} and \textit{REVISE}, but not \textit{Schut} which is penalty-free. \textit{Schut} is also the only generator that uses JSMA instead of SGD for optimization.
+
+\import{contents/}{table_params.tex}
+
+\import{contents/}{table_perf.tex}
+
+\import{contents/}{table_gen_params.tex}
+
+\subsection{Compute}
+
+To enable others to easily replicate our experiments, we have chosen to work with small neural network architectures and randomly undersampled the \textit{MNIST} dataset (maintaining class balance). All of our final benchmarks could then be run locally on a personal machine, but grid searches and the sheer number of datasets required us to move to high-performance computing clusters to conduct our experiments efficiently.
+
+\subsubsection{Local runs}
+
+The longest runtimes we experienced for model training and counterfactual benchmarking for a single benchmark were on the order of 12-24 hours (\textit{MNIST} data). For the synthetic data, single benchmarks could be completed in less than an hour. We have summarised our system information below:
+
+\textbf{Software}:
+
+\begin{itemize}
+ \item System Version: macOS 13.3.1
+ \item Kernel Version: Darwin 22.4.0
+\end{itemize}
+
+\textbf{Hardware}:
+
+\begin{itemize}
+ \item Model Name: MacBook Pro
+ \item Model Identifier: MacBookPro16,1
+ \item Processor Name: 8-Core Intel Core i9
+ \item Processor Speed: 2.3 GHz
+ \item Number of Processors: 1
+ \item Total Number of Cores: 8
+ \item L2 Cache (per Core): 256 KB
+ \item L3 Cache: 16 MB
+ \item Hyper-Threading Technology: Enabled
+ \item Memory: 32 GB
+\end{itemize}
+
+\subsubsection{HPC}
+
+We used two large clusters, which we do not cite or mention by name at this point to not interfer with the double-blind review process. All of our grid searches were multi-processed on 100 CPUs at 4-8GB or memory each.
+
+\subsection{Results}\label{app:results}
+
+Figures~\ref{fig:mnist-eccco-lenet} to~\ref{fig:mnist-eccco-jem-ens} show examples of counterfactuals for \textit{MNIST} generated by \textit{ECCCo+} for our different models. Original images are shown on the diagonal and the corresponding counterfactuals are plotted across rows. Figures~\ref{fig:mnist-revise-lenet} to~\ref{fig:mnist-revise-jem-ens} show the same examples but for \textit{REVISE}. Both counterfactual generators have access to the same optimizer. While the results for \textit{REVISE} look fairly poor here, we have observed better results for optizers with higher step sizes. Note that the seemingly poor performance by \textit{REVISE} upon visual inspection is not driven by a weak surrogate VAE: Figure~\ref{fig:vae-reg} shows image reconstructions generated by the VAE.
+
+% ECCCo params λ=[0.1,0.5,1.0], reg_strength=0.0
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_lenet_eccco.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{ECCCo+}. The underlying model is a LeNet-5 \textit{CNN}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco-lenet}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_mlp_eccco.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{ECCCo+}. The underlying model is an \textit{MLP}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco-mlp}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_mlp_ens_eccco.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{ECCCo+}. The underlying model is an \textit{MLP} ensemble. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco-mlp-ens}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_jem_eccco.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{ECCCo+}. The underlying model is a \textit{JEM}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco-jem}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_jem_ens_eccco.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{ECCCo+}. The underlying model is a \textit{JEM} ensemble. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco-jem-ens}
+\end{figure}
+
+% REVISE
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_lenet_revise.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{REVISE}. The underlying model is a LeNet-5 \textit{CNN}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-revise-lenet}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_mlp_revise.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{REVISE}. The underlying model is an \textit{MLP}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-revise-mlp}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_mlp_ens_revise.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{REVISE}. The underlying model is an \textit{MLP} ensemble. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-revise-mlp-ens}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_jem_revise.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{REVISE}. The underlying model is a \textit{JEM}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-revise-jem}
+\end{figure}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_all_jem_ens_revise.png}
+ \caption{Counterfactuals for \textit{MNIST} data generated by \textit{REVISE}. The underlying model is a \textit{JEM} ensemble. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-revise-jem-ens}
+\end{figure}
+
+\begin{figure}[h]
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/vae_rec.png}
+ \caption{Randomly drawn \textit{MNIST} images and their reconstructions generated by the VAE used by \textit{REVISE}.}\label{fig:vae-rec}
+\end{figure}
+
+Tables~\ref{tab:results-linearly-separable} to~\ref{tab:results-fashion-mnist} reports all of the evaluation metrics we have computed. Tables~\ref{tab:results-linearly-separable-valid} to~\ref{tab:results-fashion-mnist-valid} reports the same metrics for the subset of valid counterfactuals. The `Unfaithfulness' and `Implausibility' metrics have been discussed extensively in the body of the paper. The `Cost' metric relates to the distance between the factual and the counterfactual and is measured using the L1 Norm. The `Redundancy' metric measures sparsity in is defined as the percentage of features that remain unperturbed (higher is better). The `Uncertainty' metric is just the average value of the smooth set size penalty (Equation~\ref{eq:setsize}). Finally, `Validity' is the percentage of valid counterfactuals.
+
+\import{contents/}{table-linearly-separable.tex}
+
+\import{contents/}{table-circles.tex}
+
+\import{contents/}{table-moons.tex}
+
+\import{contents/}{table-california-housing.tex}
+
+\import{contents/}{table-gmsc.tex}
+
+\import{contents/}{table-german-credit.tex}
+
+\import{contents/}{table-mnist.tex}
+
+\import{contents/}{table-fashion-mnist.tex}
+
+\import{contents/}{table-linearly-separable-valid.tex}
+
+\import{contents/}{table-circles-valid.tex}
+
+\import{contents/}{table-moons-valid.tex}
+
+\import{contents/}{table-california-housing-valid.tex}
+
+\import{contents/}{table-gmsc-valid.tex}
+
+\import{contents/}{table-german-credit-valid.tex}
+
+\import{contents/}{table-mnist-valid.tex}
+
+\import{contents/}{table-fashion-mnist-valid.tex}
\ No newline at end of file
diff --git a/paper/bib.bib b/paper/bib.bib
index daed3cba558bf28a0516e5454625b5b48ea016e8..c660ab87251582c320befbf03f39b40e276aa965 100644
--- a/paper/bib.bib
+++ b/paper/bib.bib
@@ -1,33 +1,30 @@
-@TechReport{kingma2017adam,
- author = {Kingma, Diederik P. and Ba, Jimmy},
- date = {2017-01},
+@TechReport{xu2022conformal,
+ author = {Xu, Chen and Xie, Yao},
+ date = {2022-06},
institution = {arXiv},
- title = {Adam: {A} {Method} for {Stochastic} {Optimization}},
- doi = {10.48550/arXiv.1412.6980},
- note = {arXiv:1412.6980 [cs] type: article},
- url = {http://arxiv.org/abs/1412.6980},
- urldate = {2023-05-17},
- abstract = {We introduce Adam, an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. The method is straightforward to implement, is computationally efficient, has little memory requirements, is invariant to diagonal rescaling of the gradients, and is well suited for problems that are large in terms of data and/or parameters. The method is also appropriate for non-stationary objectives and problems with very noisy and/or sparse gradients. The hyper-parameters have intuitive interpretations and typically require little tuning. Some connections to related algorithms, on which Adam was inspired, are discussed. We also analyze the theoretical convergence properties of the algorithm and provide a regret bound on the convergence rate that is comparable to the best known results under the online convex optimization framework. Empirical results demonstrate that Adam works well in practice and compares favorably to other stochastic optimization methods. Finally, we discuss AdaMax, a variant of Adam based on the infinity norm.},
- annotation = {Comment: Published as a conference paper at the 3rd International Conference for Learning Representations, San Diego, 2015},
- file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1412.6980.pdf:application/pdf},
- keywords = {Computer Science - Machine Learning},
- shorttitle = {Adam},
+ title = {Conformal prediction set for time-series},
+ doi = {10.48550/arXiv.2206.07851},
+ note = {arXiv:2206.07851 [cs, stat] type: article},
+ url = {http://arxiv.org/abs/2206.07851},
+ urldate = {2023-07-22},
+ abstract = {When building either prediction intervals for regression (with real-valued response) or prediction sets for classification (with categorical responses), uncertainty quantification is essential to studying complex machine learning methods. In this paper, we develop Ensemble Regularized Adaptive Prediction Set (ERAPS) to construct prediction sets for time-series (with categorical responses), based on the prior work of [Xu and Xie, 2021]. In particular, we allow unknown dependencies to exist within features and responses that arrive in sequence. Method-wise, ERAPS is a distribution-free and ensemble-based framework that is applicable for arbitrary classifiers. Theoretically, we bound the coverage gap without assuming data exchangeability and show asymptotic set convergence. Empirically, we demonstrate valid marginal and conditional coverage by ERAPS, which also tends to yield smaller prediction sets than competing methods.},
+ annotation = {Comment: Strongly accepted by the Workshop on Distribution-Free Uncertainty Quantification at ICML 2022},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2206.07851.pdf:application/pdf},
+ keywords = {Statistics - Machine Learning, Computer Science - Machine Learning, Statistics - Methodology},
}
-@TechReport{xiao2017fashion,
- author = {Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
- date = {2017-09},
- institution = {arXiv},
- title = {Fashion-{MNIST}: a {Novel} {Image} {Dataset} for {Benchmarking} {Machine} {Learning} {Algorithms}},
- doi = {10.48550/arXiv.1708.07747},
- note = {arXiv:1708.07747 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1708.07747},
- urldate = {2023-05-10},
- abstract = {We present Fashion-MNIST, a new dataset comprising of 28x28 grayscale images of 70,000 fashion products from 10 categories, with 7,000 images per category. The training set has 60,000 images and the test set has 10,000 images. Fashion-MNIST is intended to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms, as it shares the same image size, data format and the structure of training and testing splits. The dataset is freely available at https://github.com/zalandoresearch/fashion-mnist},
- annotation = {Comment: Dataset is freely available at https://github.com/zalandoresearch/fashion-mnist Benchmark is available at http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/},
- file = {:xiao2017fashion - Fashion MNIST_ a Novel Image Dataset for Benchmarking Machine Learning Algorithms.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Statistics - Machine Learning},
- shorttitle = {Fashion-{MNIST}},
+@Article{kingma2014adam,
+ author = {Kingma, Diederik P and Ba, Jimmy},
+ title = {Adam: A method for stochastic optimization},
+ journal = {arXiv preprint arXiv:1412.6980},
+ year = {2014},
+}
+
+@Article{xiao2017fashion,
+ author = {Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
+ title = {Fashion-mnist: a novel image dataset for benchmarking machine learning algorithms},
+ journal = {arXiv preprint arXiv:1708.07747},
+ year = {2017},
}
@Online{mw2023fidelity,
@@ -41,17 +38,18 @@
}
@InProceedings{altmeyer2023endogenous,
- author = {Altmeyer, Patrick and Angela, Giovan and Buszydlik, Aleksander and Dobiczek, Karol and van Deursen, Arie and Liem, Cynthia},
- booktitle = {First {IEEE} {Conference} on {Secure} and {Trustworthy} {Machine} {Learning}},
- title = {Endogenous {Macrodynamics} in {Algorithmic} {Recourse}},
- file = {:altmeyerendogenous - Endogenous Macrodynamics in Algorithmic Recourse.pdf:PDF},
- year = {2023},
+ author = {Altmeyer, Patrick and Angela, Giovan and Buszydlik, Aleksander and Dobiczek, Karol and van Deursen, Arie and Liem, Cynthia CS},
+ booktitle = {2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)},
+ title = {Endogenous Macrodynamics in Algorithmic Recourse},
+ organization = {IEEE},
+ pages = {418--431},
+ year = {2023},
}
%% This BibTeX bibliography file was created using BibDesk.
%% https://bibdesk.sourceforge.io/
-%% Created for Anonymous Author at 2022-12-13 12:58:22 +0100
+%% Created for Patrick Altmeyer at 2022-12-13 12:58:22 +0100
%% Saved with string encoding Unicode (UTF-8)
@@ -61,6 +59,7 @@
@Article{abadie2002instrumental,
author = {Abadie, Alberto and Angrist, Joshua and Imbens, Guido},
title = {Instrumental Variables Estimates of the Effect of Subsidized Training on the Quantiles of Trainee Earnings},
+ doi = {10.2139/ssrn.195733},
number = {1},
pages = {91--117},
volume = {70},
@@ -87,6 +86,7 @@
author = {Ackerman, Samuel and Dube, Parijat and Farchi, Eitan and Raz, Orna and Zalmanovici, Marcel},
booktitle = {2021 {{IEEE}}/{{ACM Third International Workshop}} on {{Deep Learning}} for {{Testing}} and {{Testing}} for {{Deep Learning}} ({{DeepTest}})},
title = {Machine {{Learning Model Drift Detection Via Weak Data Slices}}},
+ doi = {10.1109/deeptest52559.2021.00007},
pages = {1--8},
publisher = {{IEEE}},
date-added = {2022-12-13 12:58:01 +0100},
@@ -97,6 +97,7 @@
@Article{allen2017referencedependent,
author = {Allen, Eric J and Dechow, Patricia M and Pope, Devin G and Wu, George},
title = {Reference-Dependent Preferences: {{Evidence}} from Marathon Runners},
+ doi = {10.3386/w20343},
number = {6},
pages = {1657--1672},
volume = {63},
@@ -153,16 +154,11 @@
year = {2022},
}
-@Unpublished{angelopoulos2021gentle,
- author = {Angelopoulos, Anastasios N. and Bates, Stephen},
- title = {A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2107.07511},
- eprinttype = {arxiv},
- file = {:/Users/FA31DU/Zotero/storage/RKSUMYZG/Angelopoulos and Bates - 2021 - A gentle introduction to conformal prediction and .pdf:;:/Users/FA31DU/Zotero/storage/PRUEKRR3/2107.html:},
- year = {2021},
+@Article{angelopoulos2021gentle,
+ author = {Angelopoulos, Anastasios N and Bates, Stephen},
+ title = {A gentle introduction to conformal prediction and distribution-free uncertainty quantification},
+ journal = {arXiv preprint arXiv:2107.07511},
+ year = {2021},
}
@Misc{angelopoulos2022uncertainty,
@@ -187,6 +183,7 @@
@Article{angelucci2009indirect,
author = {Angelucci, Manuela and De Giorgi, Giacomo},
title = {Indirect Effects of an Aid Program: How Do Cash Transfers Affect Ineligibles' Consumption?},
+ doi = {10.1257/aer.99.1.486},
number = {1},
pages = {486--508},
volume = {99},
@@ -230,6 +227,7 @@
@Article{ariely2003coherent,
author = {Ariely, Dan and Loewenstein, George and Prelec, Drazen},
title = {``{{Coherent}} Arbitrariness'': {{Stable}} Demand Curves without Stable Preferences},
+ doi = {10.1017/cbo9780511618031.014},
number = {1},
pages = {73--106},
volume = {118},
@@ -242,6 +240,7 @@
@Article{ariely2006tom,
author = {Ariely, Dan and Loewenstein, George and Prelec, Drazen},
title = {Tom {{Sawyer}} and the Construction of Value},
+ doi = {10.1017/cbo9780511618031.015},
number = {1},
pages = {1--10},
volume = {60},
@@ -254,6 +253,7 @@
@Article{arrieta2020explainable,
author = {Arrieta, Alejandro Barredo and Diaz-Rodriguez, Natalia and Del Ser, Javier and Bennetot, Adrien and Tabik, Siham and Barbado, Alberto and Garcia, Salvador and Gil-Lopez, Sergio and Molina, Daniel and Benjamins, Richard and others},
title = {Explainable {{Artificial Intelligence}} ({{XAI}}): {{Concepts}}, Taxonomies, Opportunities and Challenges toward Responsible {{AI}}},
+ doi = {10.1016/j.inffus.2019.12.012},
pages = {82--115},
volume = {58},
date-added = {2022-12-13 12:58:01 +0100},
@@ -297,6 +297,7 @@
@Article{bechara1997deciding,
author = {Bechara, Antoine and Damasio, Hanna and Tranel, Daniel and Damasio, Antonio R},
title = {Deciding Advantageously before Knowing the Advantageous Strategy},
+ doi = {10.7551/mitpress/3077.003.0044},
number = {5304},
pages = {1293--1295},
volume = {275},
@@ -310,6 +311,7 @@
@Book{berlinet2011reproducing,
author = {Berlinet, Alain and Thomas-Agnan, Christine},
title = {Reproducing Kernel {{Hilbert}} Spaces in Probability and Statistics},
+ doi = {10.1007/978-1-4419-9096-9},
publisher = {{Springer Science \& Business Media}},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -319,6 +321,7 @@
@Misc{bernanke1990federal,
author = {Bernanke, Ben S},
title = {The Federal Funds Rate and the Channels of Monetary Transnission},
+ doi = {10.3386/w3487},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
publisher = {{National Bureau of Economic Research Cambridge, Mass., USA}},
@@ -390,6 +393,7 @@
@Article{borch2022machine,
author = {Borch, Christian},
title = {Machine Learning, Knowledge Risk, and Principal-Agent Problems in Automated Trading},
+ doi = {10.1016/j.techsoc.2021.101852},
pages = {101852},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -397,20 +401,18 @@
year = {2022},
}
-@Unpublished{borisov2021deep,
- author = {Borisov, Vadim and Leemann, Tobias and Se{\ss}ler, Kathrin and Haug, Johannes and Pawelczyk, Martin and Kasneci, Gjergji},
- title = {Deep Neural Networks and Tabular Data: {{A}} Survey},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2110.01889},
- eprinttype = {arxiv},
- year = {2021},
+@Article{borisov2022deep,
+ author = {Borisov, Vadim and Leemann, Tobias and Se{\ss}ler, Kathrin and Haug, Johannes and Pawelczyk, Martin and Kasneci, Gjergji},
+ title = {Deep neural networks and tabular data: A survey},
+ journal = {IEEE Transactions on Neural Networks and Learning Systems},
+ publisher = {IEEE},
+ year = {2022},
}
@Article{bramoulle2009identification,
author = {Bramoull{\'e}, Yann and Djebbari, Habiba and Fortin, Bernard},
title = {Identification of Peer Effects through Social Networks},
+ doi = {10.2139/ssrn.965818},
number = {1},
pages = {41--55},
volume = {150},
@@ -423,6 +425,7 @@
@Article{bramoulle2020peer,
author = {Bramoull{\'e}, Yann and Djebbari, Habiba and Fortin, Bernard},
title = {Peer Effects in Networks: {{A}} Survey},
+ doi = {10.2139/ssrn.3534495},
pages = {603--629},
volume = {12},
date-added = {2022-12-13 12:58:01 +0100},
@@ -445,6 +448,7 @@
@Book{brock1991nonlinear,
author = {Brock, William Allen and Brock, William A and Hsieh, David Arthur and LeBaron, Blake Dean and Brock, William E},
title = {Nonlinear Dynamics, Chaos, and Instability: Statistical Theory and Economic Evidence},
+ doi = {10.2307/2234554},
publisher = {{MIT press}},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -476,6 +480,7 @@
@Report{card1993minimum,
author = {Card, David and Krueger, Alan B},
title = {Minimum Wages and Employment: {{A}} Case Study of the Fast Food Industry in {{New Jersey}} and {{Pennsylvania}}},
+ doi = {10.3386/w4509},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
school = {{National Bureau of Economic Research}},
@@ -486,6 +491,7 @@
author = {Carlini, Nicholas and Wagner, David},
booktitle = {2017 Ieee Symposium on Security and Privacy (Sp)},
title = {Towards Evaluating the Robustness of Neural Networks},
+ doi = {10.1109/sp.2017.49},
pages = {39--57},
publisher = {{IEEE}},
date-added = {2022-12-13 12:58:01 +0100},
@@ -506,6 +512,7 @@
@Article{carrell2009does,
author = {Carrell, Scott E and Fullerton, Richard L and West, James E},
title = {Does Your Cohort Matter? {{Measuring}} Peer Effects in College Achievement},
+ doi = {10.3386/w14032},
number = {3},
pages = {439--464},
volume = {27},
@@ -518,6 +525,7 @@
@Article{carrell2013natural,
author = {Carrell, Scott E and Sacerdote, Bruce I and West, James E},
title = {From Natural Variation to Optimal Policy? {{The}} Importance of Endogenous Peer Group Formation},
+ doi = {10.3982/ecta10168},
number = {3},
pages = {855--882},
volume = {81},
@@ -539,6 +547,7 @@
@Article{cascarino2022explainable,
author = {Cascarino, Giuseppe and Moscatelli, Mirko and Parlapiano, Fabio},
title = {Explainable {{Artificial Intelligence}}: Interpreting Default Forecasting Models Based on {{Machine Learning}}},
+ doi = {10.2139/ssrn.4090707},
number = {674},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -572,6 +581,7 @@
@Article{chetty2011adjustment,
author = {Chetty, Raj and Friedman, John N and Olsen, Tore and Pistaferri, Luigi},
title = {Adjustment Costs, Firm Responses, and Micro vs. Macro Labor Supply Elasticities: {{Evidence}} from {{Danish}} Tax Records},
+ doi = {10.3386/w15617},
number = {2},
pages = {749--804},
volume = {126},
@@ -596,6 +606,7 @@
@Article{crawford2019variable,
author = {Crawford, Lorin and Flaxman, Seth R and Runcie, Daniel E and West, Mike},
title = {Variable Prioritization in Nonlinear Black Box Methods: {{A}} Genetic Association Case Study},
+ doi = {10.1214/18-aoas1222},
number = {2},
pages = {958},
volume = {13},
@@ -608,6 +619,7 @@
@InProceedings{dai2022counterfactual,
author = {Dai, Xinyue and Keane, Mark T and Shalloo, Laurence and Ruelle, Elodie and Byrne, Ruth MJ},
title = {Counterfactual Explanations for Prediction and Diagnosis in Xai},
+ doi = {10.1145/3514094.3534144},
eventtitle = {Proceedings of the 2022 {{AAAI}}/{{ACM Conference}} on {{AI}}, {{Ethics}}, and {{Society}}},
pages = {215--226},
date-added = {2022-12-13 12:58:01 +0100},
@@ -638,6 +650,7 @@
@Article{dehejia1999causal,
author = {Dehejia, Rajeev H and Wahba, Sadek},
title = {Causal Effects in Nonexperimental Studies: {{Reevaluating}} the Evaluation of Training Programs},
+ doi = {10.1080/01621459.1999.10473858},
number = {448},
pages = {1053--1062},
volume = {94},
@@ -650,6 +663,7 @@
@Article{dell2010persistent,
author = {Dell, Melissa},
title = {The Persistent Effects of {{Peru}}'s Mining Mita},
+ doi = {10.2139/ssrn.1596425},
number = {6},
pages = {1863--1903},
volume = {78},
@@ -663,6 +677,7 @@
@Article{denhengst2020reinforcement,
author = {den Hengst, Floris and Grua, Eoin Martino and el Hassouni, Ali and Hoogendoorn, Mark},
title = {Reinforcement Learning for Personalization: {{A}} Systematic Literature Review},
+ doi = {10.3233/ds-200028},
issue = {Preprint},
pages = {1--41},
date-added = {2022-12-13 12:58:01 +0100},
@@ -675,6 +690,7 @@
@Article{deoliveira2021framework,
author = {de Oliveira, Raphael Mazzine Barbosa and Martens, David},
title = {A Framework and Benchmarking Study for Counterfactual Generating Methods on Tabular Data},
+ doi = {10.3390/app11167274},
number = {16},
pages = {7274},
volume = {11},
@@ -685,16 +701,6 @@
year = {2021},
}
-@Article{dhurandhar2018explanations,
- author = {Dhurandhar, Amit and Chen, Pin-Yu and Luss, Ronny and Tu, Chun-Chen and Ting, Paishun and Shanmugam, Karthikeyan and Das, Payel},
- title = {Explanations Based on the Missing: {{Towards}} Contrastive Explanations with Pertinent Negatives},
- volume = {31},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- journal = {Advances in neural information processing systems},
- year = {2018},
-}
-
@InProceedings{dombrowski2021diffeomorphic,
author = {Dombrowski, Ann-Kathrin and Gerken, Jan E and Kessel, Pan},
booktitle = {{{ICML Workshop}} on {{Invertible Neural Networks}}, {{Normalizing Flows}}, and {{Explicit Likelihood Models}}},
@@ -717,6 +723,7 @@
@Article{epstein1979stability,
author = {Epstein, Seymour},
title = {The Stability of Behavior: {{I}}. {{On}} Predicting Most of the People Much of the Time.},
+ doi = {10.1037/0022-3514.37.7.1097},
number = {7},
pages = {1097},
volume = {37},
@@ -741,6 +748,7 @@
@Article{falk2006clean,
author = {Falk, Armin and Ichino, Andrea},
title = {Clean Evidence on Peer Effects},
+ doi = {10.1086/497818},
number = {1},
pages = {39--57},
volume = {24},
@@ -776,6 +784,7 @@
@Article{fehr2000cooperation,
author = {Fehr, Ernst and Gachter, Simon},
title = {Cooperation and Punishment in Public Goods Experiments},
+ doi = {10.2139/ssrn.203194},
number = {4},
pages = {980--994},
volume = {90},
@@ -832,6 +841,7 @@
@Article{galizzi2019external,
author = {Galizzi, Matteo M and Navarro-Martinez, Daniel},
title = {On the External Validity of Social Preference Games: A Systematic Lab-Field Study},
+ doi = {10.1287/mnsc.2017.2908},
number = {3},
pages = {976--1002},
volume = {65},
@@ -876,6 +886,7 @@
@Article{gilbert1998immune,
author = {Gilbert, Daniel T and Pinel, Elizabeth C and Wilson, Timothy D and Blumberg, Stephen J and Wheatley, Thalia P},
title = {Immune Neglect: A Source of Durability Bias in Affective Forecasting.},
+ doi = {10.1037/0022-3514.75.3.617},
number = {3},
pages = {617},
volume = {75},
@@ -888,6 +899,7 @@
@Article{gneezy2006uncertainty,
author = {Gneezy, Uri and List, John A and Wu, George},
title = {The Uncertainty Effect: {{When}} a Risky Prospect Is Valued Less than Its Worst Possible Outcome},
+ doi = {10.1093/qje/121.4.1283},
number = {4},
pages = {1283--1309},
volume = {121},
@@ -901,6 +913,7 @@
author = {Goan, Ethan and Fookes, Clinton},
booktitle = {Case {{Studies}} in {{Applied Bayesian Data Science}}},
title = {Bayesian {{Neural Networks}}: {{An Introduction}} and {{Survey}}},
+ doi = {10.1007/978-3-030-42553-1_3},
pages = {45--87},
publisher = {{Springer}},
date-added = {2022-12-13 12:58:01 +0100},
@@ -943,6 +956,7 @@
@Article{goodfriend2005incredible,
author = {Goodfriend, Marvin and King, Robert G},
title = {The Incredible {{Volcker}} Disinflation},
+ doi = {10.3386/w11562},
number = {5},
pages = {981--1015},
volume = {52},
@@ -955,6 +969,7 @@
@Article{graham2017econometric,
author = {Graham, Bryan S},
title = {An Econometric Model of Network Formation with Degree Heterogeneity},
+ doi = {10.1920/wp.cem.2017.0817},
number = {4},
pages = {1033--1063},
volume = {85},
@@ -977,6 +992,7 @@
@Article{grether1979economic,
author = {Grether, David M and Plott, Charles R},
title = {Economic Theory of Choice and the Preference Reversal Phenomenon},
+ doi = {10.1017/cbo9780511618031.006},
number = {4},
pages = {623--638},
volume = {69},
@@ -1000,9 +1016,11 @@
@Unpublished{griffith2020name,
author = {Griffith, Alan},
+ date = {2020-08-01},
title = {Name {{Your Friends}}, but {{Only Five}}? {{The Importance}} of {{Censoring}} in {{Peer Effects Estimates}} Using {{Social Network Data}}},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
+ doi = {10.1086/717935},
year = {2020},
}
@@ -1029,6 +1047,7 @@
author = {Gupta, Neha and Granmo, Ole-Christoffer and Agrawala, Ashok},
booktitle = {2011 10th {{International Conference}} on {{Machine Learning}} and {{Applications}} and {{Workshops}}},
title = {Thompson Sampling for Dynamic Multi-Armed Bandits},
+ doi = {10.1109/icmla.2011.144},
pages = {484--489},
publisher = {{IEEE}},
volume = {1},
@@ -1058,6 +1077,7 @@
@Article{hamzacebi2008improving,
author = {Hamza{\c c}ebi, Co{\c s}kun},
title = {Improving Artificial Neural Networks' Performance in Seasonal Time Series Forecasting},
+ doi = {10.1016/j.ins.2008.07.024},
number = {23},
pages = {4550--4559},
volume = {178},
@@ -1071,6 +1091,7 @@
author = {Hanneke, Steve},
booktitle = {Proceedings of the 24th International Conference on {{Machine}} Learning},
title = {A Bound on the Label Complexity of Agnostic Active Learning},
+ doi = {10.1145/1273496.1273541},
pages = {353--360},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -1080,6 +1101,7 @@
@Article{hansen2020virtue,
author = {Hansen, Kristian Bondo},
title = {The Virtue of Simplicity: {{On}} Machine Learning Models in Algorithmic Trading},
+ doi = {10.1177/2053951720926558},
number = {1},
pages = {2053951720926558},
volume = {7},
@@ -1112,6 +1134,7 @@
@Article{hershfield2011increasing,
author = {Hershfield, Hal E and Goldstein, Daniel G and Sharpe, William F and Fox, Jesse and Yeykelis, Leo and Carstensen, Laura L and Bailenson, Jeremy N},
title = {Increasing Saving Behavior through Age-Progressed Renderings of the Future Self},
+ doi = {10.1509/jmkr.48.spl.s23},
issue = {SPL},
pages = {S23--S37},
volume = {48},
@@ -1125,6 +1148,7 @@
author = {Ho, Tin Kam},
booktitle = {Proceedings of 3rd International Conference on Document Analysis and Recognition},
title = {Random Decision Forests},
+ doi = {10.1109/ICDAR.1995.598994},
pages = {278--282},
publisher = {{IEEE}},
volume = {1},
@@ -1148,9 +1172,11 @@
@Unpublished{hoff2021bayesoptimal,
author = {Hoff, Peter},
title = {Bayes-Optimal Prediction with Frequentist Coverage Control},
+ url = {10.3150/22-bej1484},
archiveprefix = {arXiv},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
+ doi = {10.3150/22-bej1484},
eprint = {2105.14045},
eprinttype = {arxiv},
file = {:/Users/FA31DU/Zotero/storage/IQK27WVA/Hoff - 2021 - Bayes-optimal prediction with frequentist coverage.pdf:;:/Users/FA31DU/Zotero/storage/K8EAZA25/2105.html:},
@@ -1164,6 +1190,7 @@
bdsk-url-1 = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
+ journal = {https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)},
year = {1994},
}
@@ -1191,8 +1218,10 @@
@Article{hsee1996evaluability,
author = {Hsee, Christopher K},
title = {The Evaluability Hypothesis: {{An}} Explanation for Preference Reversals between Joint and Separate Evaluations of Alternatives},
+ doi = {10.1006/obhd.1996.0077},
number = {3},
pages = {247--257},
+ url = {10.1006/obhd.1996.0077},
volume = {67},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -1203,8 +1232,10 @@
@Article{hsee2004music,
author = {Hsee, Christopher K and Rottenstreich, Yuval},
title = {Music, Pandas, and Muggers: On the Affective Psychology of Value.},
+ doi = {10.1017/cbo9780511618031.033},
number = {1},
pages = {23},
+ url = {10.1017/cbo9780511618031.033},
volume = {133},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -1215,6 +1246,7 @@
@Article{hsieh2016social,
author = {Hsieh, Chih-Sheng and Lee, Lung Fei},
title = {A Social Interactions Model with Endogenous Friendship Formation and Selectivity},
+ doi = {10.1002/jae.2426},
number = {2},
pages = {301--319},
volume = {31},
@@ -1249,6 +1281,7 @@
@Article{innes2018flux,
author = {Innes, Mike},
title = {Flux: {{Elegant}} Machine Learning with {{Julia}}},
+ doi = {10.21105/joss.00602},
number = {25},
pages = {602},
volume = {3},
@@ -1283,6 +1316,7 @@
@Article{jackson2007meeting,
author = {Jackson, Matthew O and Rogers, Brian W},
title = {Meeting Strangers and Friends of Friends: {{How}} Random Are Social Networks?},
+ doi = {10.1257/aer.97.3.890},
number = {3},
pages = {890--915},
volume = {97},
@@ -1306,6 +1340,7 @@
@Article{johansson2005failure,
author = {Johansson, Petter and Hall, Lars and Sikstr{\"o}m, Sverker and Olsson, Andreas},
title = {Failure to Detect Mismatches between Intention and Outcome in a Simple Decision Task},
+ doi = {10.1126/science.1111709},
number = {5745},
pages = {116--119},
volume = {310},
@@ -1319,6 +1354,7 @@
@Article{johnsson2021estimation,
author = {Johnsson, Ida and Moon, Hyungsik Roger},
title = {Estimation of Peer Effects in Endogenous Social Networks: {{Control}} Function Approach},
+ doi = {10.2139/ssrn.3043404},
number = {2},
pages = {328--345},
volume = {103},
@@ -1331,6 +1367,7 @@
@Article{jolliffe2003modified,
author = {Jolliffe, Ian T and Trendafilov, Nickolay T and Uddin, Mudassir},
title = {A Modified Principal Component Technique Based on the {{LASSO}}},
+ doi = {10.1198/1061860032148},
number = {3},
pages = {531--547},
volume = {12},
@@ -1348,15 +1385,11 @@
year = {2021},
}
-@Unpublished{joshi2019realistic,
- author = {Joshi, Shalmali and Koyejo, Oluwasanmi and Vijitbenjaronk, Warut and Kim, Been and Ghosh, Joydeep},
- title = {Towards Realistic Individual Recourse and Actionable Explanations in Black-Box Decision Making Systems},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1907.09615},
- eprinttype = {arxiv},
- year = {2019},
+@Article{joshi2019realistic,
+ author = {Joshi, Shalmali and Koyejo, Oluwasanmi and Vijitbenjaronk, Warut and Kim, Been and Ghosh, Joydeep},
+ title = {Towards realistic individual recourse and actionable explanations in black-box decision making systems},
+ journal = {arXiv preprint arXiv:1907.09615},
+ year = {2019},
}
@Unpublished{jospin2020handson,
@@ -1365,6 +1398,7 @@
archiveprefix = {arXiv},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
+ doi = {10.1109/mci.2022.3155327},
eprint = {2007.06823},
eprinttype = {arxiv},
year = {2020},
@@ -1402,6 +1436,7 @@
@Article{kahneman1990experimental,
author = {Kahneman, Daniel and Knetsch, Jack L and Thaler, Richard H},
title = {Experimental Tests of the Endowment Effect and the {{Coase}} Theorem},
+ doi = {10.1017/cbo9781139175197.009},
number = {6},
pages = {1325--1348},
volume = {98},
@@ -1414,6 +1449,7 @@
@Article{kahneman1992reference,
author = {Kahneman, Daniel},
title = {Reference Points, Anchors, Norms, and Mixed Feelings},
+ doi = {10.1016/0749-5978(92)90015-y},
number = {2},
pages = {296--312},
volume = {51},
@@ -1459,6 +1495,7 @@
author = {Kaur, Harmanpreet and Nori, Harsha and Jenkins, Samuel and Caruana, Rich and Wallach, Hanna and Wortman Vaughan, Jennifer},
booktitle = {Proceedings of the 2020 {{CHI}} Conference on Human Factors in Computing Systems},
title = {Interpreting Interpretability: Understanding Data Scientists' Use of Interpretability Tools for Machine Learning},
+ doi = {10.1145/3313831.3376219},
pages = {1--14},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -1488,6 +1525,7 @@
@Article{kihoro2004seasonal,
author = {Kihoro, J and Otieno, RO and Wafula, C},
title = {Seasonal Time Series Forecasting: {{A}} Comparative Study of {{ARIMA}} and {{ANN}} Models},
+ doi = {10.4314/ajst.v5i2.15330},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
year = {2004},
@@ -1502,17 +1540,6 @@
year = {2017},
}
-@Unpublished{kingma2014adam,
- author = {Kingma, Diederik P and Ba, Jimmy},
- title = {Adam: {{A}} Method for Stochastic Optimization},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {1412.6980},
- eprinttype = {arxiv},
- year = {2014},
-}
-
@Article{kirsch2019batchbald,
author = {Kirsch, Andreas and Van Amersfoort, Joost and Gal, Yarin},
title = {Batchbald: {{Efficient}} and Diverse Batch Acquisition for Deep Bayesian Active Learning},
@@ -1530,6 +1557,7 @@
archiveprefix = {arXiv},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
+ doi = {10.1007/978-3-030-93842-0_6},
eprint = {2111.02244},
eprinttype = {arxiv},
year = {2021},
@@ -1610,17 +1638,16 @@
}
@Article{lecun1998mnist,
- author = {LeCun, Yann},
- title = {The {{MNIST}} Database of Handwritten Digits},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- shortjournal = {http://yann. lecun. com/exdb/mnist/},
- year = {1998},
+ author = {LeCun, Yann},
+ title = {The MNIST database of handwritten digits},
+ journal = {http://yann. lecun. com/exdb/mnist/},
+ year = {1998},
}
@Article{lee2003best,
author = {Lee, Lung-fei},
title = {Best Spatial Two-Stage Least Squares Estimators for a Spatial Autoregressive Model with Autoregressive Disturbances},
+ doi = {10.1081/etc-120025891},
number = {4},
pages = {307--335},
volume = {22},
@@ -1633,6 +1660,7 @@
@Article{lerner2013financial,
author = {Lerner, Jennifer S and Li, Ye and Weber, Elke U},
title = {The Financial Costs of Sadness},
+ doi = {10.1177/0956797612450302},
number = {1},
pages = {72--79},
volume = {24},
@@ -1645,6 +1673,7 @@
@Article{list2004neoclassical,
author = {List, John A},
title = {Neoclassical Theory versus Prospect Theory: {{Evidence}} from the Marketplace},
+ doi = {10.3386/w9736},
number = {2},
pages = {615--625},
volume = {72},
@@ -1685,6 +1714,7 @@
@Article{madrian2001power,
author = {Madrian, Brigitte C and Shea, Dennis F},
title = {The Power of Suggestion: {{Inertia}} in 401 (k) Participation and Savings Behavior},
+ doi = {10.3386/w7682},
number = {4},
pages = {1149--1187},
volume = {116},
@@ -1703,15 +1733,22 @@
year = {2008},
}
-@misc{manokhin2022awesome,
- author = {Manokhin, Valery},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- title = {Awesome Conformal Prediction}}
+@Software{manokhin2022awesome,
+ author = {Manokhin, Valery},
+ title = {Awesome Conformal Prediction},
+ doi = {10.5281/zenodo.6467205},
+ note = {{"If you use Awesome Conformal Prediction, please cite it as below."}},
+ url = {https://doi.org/10.5281/zenodo.6467205},
+ version = {v1.0.0},
+ month = apr,
+ publisher = {Zenodo},
+ year = {2022},
+}
@Article{manski1993identification,
author = {Manski, Charles F},
title = {Identification of Endogenous Social Effects: {{The}} Reflection Problem},
+ doi = {10.2307/2298123},
number = {3},
pages = {531--542},
volume = {60},
@@ -1724,6 +1761,7 @@
@Article{markle2018goals,
author = {Markle, Alex and Wu, George and White, Rebecca and Sackett, Aaron},
title = {Goals as Reference Points in Marathon Running: {{A}} Novel Test of Reference Dependence},
+ doi = {10.2139/ssrn.2523510},
number = {1},
pages = {19--50},
volume = {56},
@@ -1745,6 +1783,7 @@
@Article{mccracken2016fredmd,
author = {McCracken, Michael W and Ng, Serena},
title = {{{FRED-MD}}: {{A}} Monthly Database for Macroeconomic Research},
+ doi = {10.20955/wp.2015.012},
number = {4},
pages = {574--589},
volume = {34},
@@ -1769,6 +1808,7 @@
@Article{migut2015visualizing,
author = {Migut, MA and Worring, Marcel and Veenman, Cor J},
title = {Visualizing Multi-Dimensional Decision Boundaries in {{2D}}},
+ doi = {10.1007/s10618-013-0342-x},
number = {1},
pages = {273--295},
volume = {29},
@@ -1781,6 +1821,7 @@
@Article{miller2019explanation,
author = {Miller, Tim},
title = {Explanation in Artificial Intelligence: {{Insights}} from the Social Sciences},
+ doi = {10.1016/j.artint.2018.07.007},
pages = {1--38},
volume = {267},
date-added = {2022-12-13 12:58:01 +0100},
@@ -1812,6 +1853,7 @@
@Article{mischel1988nature,
author = {Mischel, Walter and Shoda, Yuichi and Peake, Philip K},
title = {The Nature of Adolescent Competencies Predicted by Preschool Delay of Gratification.},
+ doi = {10.1037/0022-3514.54.4.687},
number = {4},
pages = {687},
volume = {54},
@@ -1825,19 +1867,20 @@
author = {Mittelstadt, Brent and Russell, Chris and Wachter, Sandra},
booktitle = {Proceedings of the Conference on Fairness, Accountability, and Transparency},
title = {Explaining Explanations in {{AI}}},
+ doi = {10.1145/3287560.3287574},
pages = {279--288},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
year = {2019},
}
-@Book{molnar2020interpretable,
- author = {Molnar, Christoph},
- title = {Interpretable Machine Learning},
- publisher = {{Lulu. com}},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- year = {2020},
+@Book{molnar2022interpretable,
+ author = {Christoph Molnar},
+ title = {Interpretable Machine Learning},
+ edition = {2},
+ subtitle = {A Guide for Making Black Box Models Explainable},
+ url = {https://christophm.github.io/interpretable-ml-book},
+ year = {2022},
}
@Book{morgan2015counterfactuals,
@@ -1865,6 +1908,7 @@
author = {Mothilal, Ramaravind K and Sharma, Amit and Tan, Chenhao},
booktitle = {Proceedings of the 2020 {{Conference}} on {{Fairness}}, {{Accountability}}, and {{Transparency}}},
title = {Explaining Machine Learning Classifiers through Diverse Counterfactual Explanations},
+ doi = {10.1145/3351095.3372850},
pages = {607--617},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -1922,6 +1966,7 @@
author = {Nelson, Kevin and Corbin, George and Anania, Mark and Kovacs, Matthew and Tobias, Jeremy and Blowers, Misty},
booktitle = {2015 {{IEEE Symposium}} on {{Computational Intelligence}} for {{Security}} and {{Defense Applications}} ({{CISDA}})},
title = {Evaluating Model Drift in Machine Learning Algorithms},
+ doi = {10.1109/cisda.2015.7208643},
pages = {1--8},
publisher = {{IEEE}},
date-added = {2022-12-13 12:58:01 +0100},
@@ -1971,6 +2016,7 @@
@Article{pace1997sparse,
author = {Pace, R Kelley and Barry, Ronald},
title = {Sparse Spatial Autoregressions},
+ doi = {10.1016/s0167-7152(96)00140-x},
number = {3},
pages = {291--297},
volume = {33},
@@ -2014,6 +2060,7 @@
@Article{pearl2019seven,
author = {Pearl, Judea},
title = {The Seven Tools of Causal Inference, with Reflections on Machine Learning},
+ doi = {10.1145/3241036},
number = {3},
pages = {54--60},
volume = {62},
@@ -2037,6 +2084,7 @@
@Book{perry2010economic,
author = {Perry, George L and Tobin, James},
title = {Economic {{Events}}, {{Ideas}}, and {{Policies}}: The 1960s and After},
+ doi = {10.5860/choice.38-4002},
publisher = {{Brookings Institution Press}},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -2089,6 +2137,7 @@
@Article{qu2015estimating,
author = {Qu, Xi and Lee, Lung-fei},
title = {Estimating a Spatial Autoregressive Model with an Endogenous Spatial Weight Matrix},
+ doi = {10.1016/j.jeconom.2014.08.008},
number = {2},
pages = {209--232},
volume = {184},
@@ -2144,7 +2193,7 @@
@InProceedings{ribeiro2016why,
author = {Ribeiro, Marco Tulio and Singh, Sameer and Guestrin, Carlos},
booktitle = {Proceedings of the 22nd {{ACM SIGKDD}} International Conference on Knowledge Discovery and Data Mining},
- title = {"{{Why}} Should i Trust You?" {{Explaining}} the Predictions of Any Classifier},
+ title = {"{{Why}} Should I Trust You?" {{Explaining}} the Predictions of Any Classifier},
pages = {1135--1144},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -2154,6 +2203,7 @@
@Article{romer1989does,
author = {Romer, Christina D and Romer, David H},
title = {Does Monetary Policy Matter? {{A}} New Test in the Spirit of {{Friedman}} and {{Schwartz}}},
+ doi = {10.3386/w2966},
pages = {121--170},
volume = {4},
date-added = {2022-12-13 12:58:01 +0100},
@@ -2165,6 +2215,7 @@
@Article{rudin2019stop,
author = {Rudin, Cynthia},
title = {Stop Explaining Black Box Machine Learning Models for High Stakes Decisions and Use Interpretable Models Instead},
+ doi = {10.1038/s42256-019-0048-x},
number = {5},
pages = {206--215},
volume = {1},
@@ -2177,6 +2228,7 @@
@Article{sacerdote2001peer,
author = {Sacerdote, Bruce},
title = {Peer Effects with Random Assignment: {{Results}} for {{Dartmouth}} Roommates},
+ doi = {10.3386/w7469},
number = {2},
pages = {681--704},
volume = {116},
@@ -2189,6 +2241,7 @@
@Article{sadinle2019least,
author = {Sadinle, Mauricio and Lei, Jing and Wasserman, Larry},
title = {Least Ambiguous Set-Valued Classifiers with Bounded Error Levels},
+ doi = {10.1080/01621459.2017.1395341},
number = {525},
pages = {223--234},
volume = {114},
@@ -2204,6 +2257,7 @@
author = {Satopaa, Ville and Albrecht, Jeannie and Irwin, David and Raghavan, Barath},
booktitle = {2011 31st International Conference on Distributed Computing Systems Workshops},
title = {Finding a" Kneedle" in a Haystack: {{Detecting}} Knee Points in System Behavior},
+ doi = {10.1109/icdcsw.2011.20},
pages = {166--171},
publisher = {{IEEE}},
date-added = {2022-12-13 12:58:01 +0100},
@@ -2247,6 +2301,7 @@
@Article{simonson1989choice,
author = {Simonson, Itamar},
title = {Choice Based on Reasons: {{The}} Case of Attraction and Compromise Effects},
+ doi = {10.1086/209205},
number = {2},
pages = {158--174},
volume = {16},
@@ -2259,6 +2314,7 @@
@Article{sims1986are,
author = {Sims, Christopher A and others},
title = {Are Forecasting Models Usable for Policy Analysis?},
+ doi = {10.21034/qr.1011},
issue = {Win},
pages = {2--16},
volume = {10},
@@ -2300,16 +2356,11 @@
year = {1974},
}
-@Unpublished{spooner2021counterfactual,
- author = {Spooner, Thomas and Dervovic, Danial and Long, Jason and Shepard, Jon and Chen, Jiahao and Magazzeni, Daniele},
- title = {Counterfactual {{Explanations}} for {{Arbitrary Regression Models}}},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2106.15212},
- eprinttype = {arxiv},
- shortjournal = {arXiv preprint arXiv:2106.15212},
- year = {2021},
+@Article{spooner2021counterfactual,
+ author = {Spooner, Thomas and Dervovic, Danial and Long, Jason and Shepard, Jon and Chen, Jiahao and Magazzeni, Daniele},
+ title = {Counterfactual explanations for arbitrary regression models},
+ journal = {arXiv preprint arXiv:2106.15212},
+ year = {2021},
}
@Article{srivastava2014dropout,
@@ -2339,6 +2390,7 @@
@Article{sturm2014simple,
author = {Sturm, Bob L},
title = {A Simple Method to Determine If a Music Information Retrieval System Is a ``Horse''},
+ doi = {10.1109/tmm.2014.2330697},
number = {6},
pages = {1636--1644},
volume = {16},
@@ -2351,6 +2403,7 @@
@Article{sunstein2003libertarian,
author = {Sunstein, Cass R and Thaler, Richard H},
title = {Libertarian Paternalism Is Not an Oxymoron},
+ doi = {10.1017/cbo9780511618031.039},
pages = {1159--1202},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -2381,6 +2434,7 @@
@Article{thaler1981empirical,
author = {Thaler, Richard},
title = {Some Empirical Evidence on Dynamic Inconsistency},
+ doi = {10.1016/0165-1765(81)90067-7},
number = {3},
pages = {201--207},
volume = {8},
@@ -2393,6 +2447,7 @@
@Article{thaler2004more,
author = {Thaler, Richard H and Benartzi, Shlomo},
title = {Save More Tomorrow{\texttrademark}: {{Using}} Behavioral Economics to Increase Employee Saving},
+ doi = {10.1086/380085},
number = {S1},
pages = {S164--S187},
volume = {112},
@@ -2405,6 +2460,7 @@
@Article{tversky1981framing,
author = {Tversky, Amos and Kahneman, Daniel},
title = {The Framing of Decisions and the Psychology of Choice},
+ doi = {10.1007/978-1-4613-2391-4_2},
number = {4481},
pages = {453--458},
volume = {211},
@@ -2418,6 +2474,7 @@
@Article{ungemach2011how,
author = {Ungemach, Christoph and Stewart, Neil and Reimers, Stian},
title = {How Incidental Values from the Environment Affect Decisions about Money, Risk, and Delay},
+ doi = {10.1177/0956797610396225},
number = {2},
pages = {253--260},
volume = {22},
@@ -2442,6 +2499,7 @@
author = {Ustun, Berk and Spangher, Alexander and Liu, Yang},
booktitle = {Proceedings of the {{Conference}} on {{Fairness}}, {{Accountability}}, and {{Transparency}}},
title = {Actionable Recourse in Linear Classification},
+ doi = {10.1145/3287560.3287566},
pages = {10--19},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
@@ -2451,6 +2509,7 @@
@Article{vanboven2000egocentric,
author = {Van Boven, Leaf and Dunning, David and Loewenstein, George},
title = {Egocentric Empathy Gaps between Owners and Buyers: Misperceptions of the Endowment Effect.},
+ doi = {10.1037/0022-3514.79.1.66},
number = {1},
pages = {66},
volume = {79},
@@ -2484,6 +2543,7 @@
@Article{verstyuk2020modeling,
author = {Verstyuk, Sergiy},
title = {Modeling Multivariate Time Series in Economics: {{From}} Auto-Regressions to Recurrent Neural Networks},
+ doi = {10.2139/ssrn.3589337},
date-added = {2022-12-13 12:58:01 +0100},
date-modified = {2022-12-13 12:58:01 +0100},
journal = {Available at SSRN 3589337},
@@ -2493,6 +2553,7 @@
@Article{wachter2017counterfactual,
author = {Wachter, Sandra and Mittelstadt, Brent and Russell, Chris},
title = {Counterfactual Explanations without Opening the Black Box: {{Automated}} Decisions and the {{GDPR}}},
+ doi = {10.2139/ssrn.3063289},
pages = {841},
volume = {31},
date-added = {2022-12-13 12:58:01 +0100},
@@ -2534,6 +2595,7 @@
@Article{widmer1996learning,
author = {Widmer, Gerhard and Kubat, Miroslav},
title = {Learning in the Presence of Concept Drift and Hidden Contexts},
+ doi = {10.1007/bf00116900},
number = {1},
pages = {69--101},
volume = {23},
@@ -2543,20 +2605,17 @@
year = {1996},
}
-@Unpublished{wilson2020case,
- author = {Wilson, Andrew Gordon},
- title = {The Case for {{Bayesian}} Deep Learning},
- archiveprefix = {arXiv},
- date-added = {2022-12-13 12:58:01 +0100},
- date-modified = {2022-12-13 12:58:01 +0100},
- eprint = {2001.10995},
- eprinttype = {arxiv},
- year = {2020},
+@Article{wilson2020case,
+ author = {Wilson, Andrew Gordon},
+ title = {The case for Bayesian deep learning},
+ journal = {arXiv preprint arXiv:2001.10995},
+ year = {2020},
}
@Article{witten2009penalized,
author = {Witten, Daniela M and Tibshirani, Robert and Hastie, Trevor},
title = {A Penalized Matrix Decomposition, with Applications to Sparse Principal Components and Canonical Correlation Analysis},
+ doi = {10.1093/biostatistics/kxp008},
number = {3},
pages = {515--534},
volume = {10},
@@ -2583,6 +2642,7 @@
@Article{yeh2009comparisons,
author = {Yeh, I-Cheng and Lien, Che-hui},
title = {The Comparisons of Data Mining Techniques for the Predictive Accuracy of Probability of Default of Credit Card Clients},
+ doi = {10.1016/j.eswa.2007.12.020},
number = {2},
pages = {2473--2480},
volume = {36},
@@ -2607,6 +2667,7 @@
@Article{zhang2003time,
author = {Zhang, G Peter},
title = {Time Series Forecasting Using a Hybrid {{ARIMA}} and Neural Network Model},
+ doi = {10.1016/s0925-2312(01)00702-0},
pages = {159--175},
volume = {50},
date-added = {2022-12-13 12:58:01 +0100},
@@ -2703,56 +2764,35 @@
}
@Book{murphy2023probabilistic,
- author = {Murphy, Kevin P.},
- date = {2023},
- title = {Probabilistic machine learning: {Advanced} topics},
- publisher = {MIT Press},
- shorttitle = {Probabilistic machine learning},
+ author = {Murphy, Kevin P},
+ title = {Probabilistic machine learning: Advanced topics},
+ publisher = {MIT press},
+ year = {2023},
}
-@TechReport{artelt2021evaluating,
- author = {Artelt, André and Vaquet, Valerie and Velioglu, Riza and Hinder, Fabian and Brinkrolf, Johannes and Schilling, Malte and Hammer, Barbara},
- date = {2021-07},
- institution = {arXiv},
- title = {Evaluating {Robustness} of {Counterfactual} {Explanations}},
- note = {arXiv:2103.02354 [cs] type: article},
- url = {http://arxiv.org/abs/2103.02354},
- urldate = {2023-03-24},
- abstract = {Transparency is a fundamental requirement for decision making systems when these should be deployed in the real world. It is usually achieved by providing explanations of the system's behavior. A prominent and intuitive type of explanations are counterfactual explanations. Counterfactual explanations explain a behavior to the user by proposing actions -- as changes to the input -- that would cause a different (specified) behavior of the system. However, such explanation methods can be unstable with respect to small changes to the input -- i.e. even a small change in the input can lead to huge or arbitrary changes in the output and of the explanation. This could be problematic for counterfactual explanations, as two similar individuals might get very different explanations. Even worse, if the recommended actions differ considerably in their complexity, one would consider such unstable (counterfactual) explanations as individually unfair. In this work, we formally and empirically study the robustness of counterfactual explanations in general, as well as under different models and different kinds of perturbations. Furthermore, we propose that plausible counterfactual explanations can be used instead of closest counterfactual explanations to improve the robustness and consequently the individual fairness of counterfactual explanations.},
- annotation = {Comment: Rewrite paper to make things more clear; Remove one theorem \& corollary due to buggy proof},
- file = {:artelt2021evaluating - Evaluating Robustness of Counterfactual Explanations.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
+@InProceedings{artelt2021evaluating,
+ author = {Artelt, Andr{\'e} and Vaquet, Valerie and Velioglu, Riza and Hinder, Fabian and Brinkrolf, Johannes and Schilling, Malte and Hammer, Barbara},
+ booktitle = {2021 IEEE Symposium Series on Computational Intelligence (SSCI)},
+ title = {Evaluating robustness of counterfactual explanations},
+ organization = {IEEE},
+ pages = {01--09},
+ year = {2021},
}
@Article{guidotti2022counterfactual,
- author = {Guidotti, Riccardo},
- date = {2022-04},
- journaltitle = {Data Mining and Knowledge Discovery},
- title = {Counterfactual explanations and how to find them: literature review and benchmarking},
- doi = {10.1007/s10618-022-00831-6},
- issn = {1573-756X},
- language = {en},
- url = {https://doi.org/10.1007/s10618-022-00831-6},
- urldate = {2023-03-24},
- abstract = {Interpretable machine learning aims at unveiling the reasons behind predictions returned by uninterpretable classifiers. One of the most valuable types of explanation consists of counterfactuals. A counterfactual explanation reveals what should have been different in an instance to observe a diverse outcome. For instance, a bank customer asks for a loan that is rejected. The counterfactual explanation consists of what should have been different for the customer in order to have the loan accepted. Recently, there has been an explosion of proposals for counterfactual explainers. The aim of this work is to survey the most recent explainers returning counterfactual explanations. We categorize explainers based on the approach adopted to return the counterfactuals, and we label them according to characteristics of the method and properties of the counterfactuals returned. In addition, we visually compare the explanations, and we report quantitative benchmarking assessing minimality, actionability, stability, diversity, discriminative power, and running time. The results make evident that the current state of the art does not provide a counterfactual explainer able to guarantee all these properties simultaneously.},
- file = {Full Text PDF:https\://link.springer.com/content/pdf/10.1007%2Fs10618-022-00831-6.pdf:application/pdf},
- keywords = {Explainable AI, Counterfactual explanations, Contrastive explanations, Interpretable machine learning},
- shorttitle = {Counterfactual explanations and how to find them},
+ author = {Guidotti, Riccardo},
+ title = {Counterfactual explanations and how to find them: literature review and benchmarking},
+ pages = {1--55},
+ journal = {Data Mining and Knowledge Discovery},
+ publisher = {Springer},
+ year = {2022},
}
-@TechReport{mahajan2020preserving,
- author = {Mahajan, Divyat and Tan, Chenhao and Sharma, Amit},
- date = {2020-06},
- institution = {arXiv},
- title = {Preserving {Causal} {Constraints} in {Counterfactual} {Explanations} for {Machine} {Learning} {Classifiers}},
- doi = {10.48550/arXiv.1912.03277},
- note = {arXiv:1912.03277 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1912.03277},
- urldate = {2023-03-24},
- abstract = {To construct interpretable explanations that are consistent with the original ML model, counterfactual examples---showing how the model's output changes with small perturbations to the input---have been proposed. This paper extends the work in counterfactual explanations by addressing the challenge of feasibility of such examples. For explanations of ML models in critical domains such as healthcare and finance, counterfactual examples are useful for an end-user only to the extent that perturbation of feature inputs is feasible in the real world. We formulate the problem of feasibility as preserving causal relationships among input features and present a method that uses (partial) structural causal models to generate actionable counterfactuals. When feasibility constraints cannot be easily expressed, we consider an alternative mechanism where people can label generated CF examples on feasibility: whether it is feasible to intervene and realize the candidate CF example from the original input. To learn from this labelled feasibility data, we propose a modified variational auto encoder loss for generating CF examples that optimizes for feasibility as people interact with its output. Our experiments on Bayesian networks and the widely used ''Adult-Income'' dataset show that our proposed methods can generate counterfactual explanations that better satisfy feasibility constraints than existing methods.. Code repository can be accessed here: {\textbackslash}textit\{https://github.com/divyat09/cf-feasibility\}},
- annotation = {Comment: 2019 NeurIPS Workshop on Do the right thing: Machine learning and Causal Inference for improved decision making},
- file = {:mahajan2020preserving - Preserving Causal Constraints in Counterfactual Explanations for Machine Learning Classifiers.pdf:PDF},
- keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Statistics - Machine Learning},
+@Article{mahajan2019preserving,
+ author = {Mahajan, Divyat and Tan, Chenhao and Sharma, Amit},
+ title = {Preserving causal constraints in counterfactual explanations for machine learning classifiers},
+ journal = {arXiv preprint arXiv:1912.03277},
+ year = {2019},
}
@TechReport{antoran2023sampling,
@@ -2780,58 +2820,343 @@
}
@InProceedings{welling2011bayesian,
- author = {Welling, M. and Teh, Y.},
- date = {2011-06},
- title = {Bayesian {Learning} via {Stochastic} {Gradient} {Langevin} {Dynamics}},
- url = {https://www.semanticscholar.org/paper/Bayesian-Learning-via-Stochastic-Gradient-Langevin-Welling-Teh/aeed631d6a84100b5e9a021ec1914095c66de415},
- urldate = {2023-05-15},
- abstract = {In this paper we propose a new framework for learning from large scale datasets based on iterative learning from small mini-batches. By adding the right amount of noise to a standard stochastic gradient optimization algorithm we show that the iterates will converge to samples from the true posterior distribution as we anneal the stepsize. This seamless transition between optimization and Bayesian posterior sampling provides an inbuilt protection against overfitting. We also propose a practical method for Monte Carlo estimates of posterior statistics which monitors a "sampling threshold" and collects samples after it has been surpassed. We apply the method to three models: a mixture of Gaussians, logistic regression and ICA with natural gradients.},
- annotation = {[TLDR] This paper proposes a new framework for learning from large scale datasets based on iterative learning from small mini-batches by adding the right amount of noise to a standard stochastic gradient optimization algorithm and shows that the iterates will converge to samples from the true posterior distribution as the authors anneal the stepsize.},
- file = {:welling_bayesian_2011 - Bayesian Learning Via Stochastic Gradient Langevin Dynamics.html:URL;:welling2011bayesian - Bayesian Learning Via Stochastic Gradient Langevin Dynamics.pdf:PDF},
+ author = {Welling, Max and Teh, Yee W},
+ booktitle = {Proceedings of the 28th international conference on machine learning (ICML-11)},
+ title = {Bayesian learning via stochastic gradient Langevin dynamics},
+ organization = {Citeseer},
+ pages = {681--688},
+ year = {2011},
}
@Article{gill2010circular,
- author = {Gill, Jeff and Hangartner, Dominik},
- date = {2010},
- journaltitle = {Political Analysis},
- title = {Circular {Data} in {Political} {Science} and {How} to {Handle} {It}},
- doi = {10.1093/pan/mpq009},
- issn = {1047-1987, 1476-4989},
- language = {en},
- number = {3},
- pages = {316--336},
- url = {https://www.cambridge.org/core/journals/political-analysis/article/circular-data-in-political-science-and-how-to-handle-it/6DF2D9DA60C455E6A48FFB0FF011F747},
- urldate = {2023-05-15},
- volume = {18},
- abstract = {There has been no attention to circular (purely cyclical) data in political science research. We show that such data exist and are mishandled by models that do not take into account the inherently recycling nature of some phenomenon. Clock and calendar effects are the obvious cases, but directional data are observed as well. We describe a standard maximum likelihood regression modeling framework based on the von Mises distribution, then develop a general Bayesian regression procedure for the first time, providing an easy-to-use Metropolis-Hastings sampler for this approach. Applications include a chronographic analysis of U.S. domestic terrorism and directional party preferences in a two-dimensional ideological space for German Bundestag elections. The results demonstrate the importance of circular models to handle periodic and directional data in political science.},
- file = {Full Text PDF:https\://www.cambridge.org/core/services/aop-cambridge-core/content/view/6DF2D9DA60C455E6A48FFB0FF011F747/S1047198700012493a.pdf/div-class-title-circular-data-in-political-science-and-how-to-handle-it-div.pdf:application/pdf},
- publisher = {Cambridge University Press},
-}
-
-@InProceedings{liu2023goggle,
- author = {Liu, Tennison and Qian, Zhaozhi and Berrevoets, Jeroen and Schaar, Mihaela van der},
- date = {2023-02},
- title = {{GOGGLE}: {Generative} {Modelling} for {Tabular} {Data} by {Learning} {Relational} {Structure}},
- language = {en},
- url = {https://openreview.net/forum?id=fPVRcJqspu},
- urldate = {2023-05-15},
- abstract = {Deep generative models learn highly complex and non-linear representations to generate realistic synthetic data. While they have achieved notable success in computer vision and natural language processing, similar advances have been less demonstrable in the tabular domain. This is partially because generative modelling of tabular data entails a particular set of challenges, including heterogeneous relationships, limited number of samples, and difficulties in incorporating prior knowledge. Additionally, unlike their counterparts in image and sequence domain, deep generative models for tabular data almost exclusively employ fully-connected layers, which encode weak inductive biases about relationships between inputs. Real-world data generating processes can often be represented using relational structures, which encode sparse, heterogeneous relationships between variables. In this work, we learn and exploit relational structure underlying tabular data to better model variable dependence, and as a natural means to introduce regularization on relationships and include prior knowledge. Specifically, we introduce GOGGLE, an end-to-end message passing scheme that jointly learns the relational structure and corresponding functional relationships as the basis of generating synthetic samples. Using real-world datasets, we provide empirical evidence that the proposed method is effective in generating realistic synthetic data and exploiting domain knowledge for downstream tasks.},
- file = {Full Text PDF:https\://openreview.net/pdf?id=fPVRcJqspu:application/pdf},
- shorttitle = {{GOGGLE}},
+ author = {Gill, Jeff and Hangartner, Dominik},
+ title = {Circular data in political science and how to handle it},
+ number = {3},
+ pages = {316--336},
+ volume = {18},
+ journal = {Political Analysis},
+ publisher = {Cambridge University Press},
+ year = {2010},
+}
+
+@InProceedings{liu2022goggle,
+ author = {Liu, Tennison and Qian, Zhaozhi and Berrevoets, Jeroen and van der Schaar, Mihaela},
+ booktitle = {The Eleventh International Conference on Learning Representations},
+ title = {GOGGLE: Generative modelling for tabular data by learning relational structure},
+ year = {2022},
+}
+
+@Article{du2019implicit,
+ author = {Du, Yilun and Mordatch, Igor},
+ title = {Implicit generation and generalization in energy-based models},
+ journal = {arXiv preprint arXiv:1903.08689},
+ year = {2019},
+}
+
+@InProceedings{krizhevsky2009learning,
+ author = {Krizhevsky, A.},
+ date = {2009},
+ title = {Learning {Multiple} {Layers} of {Features} from {Tiny} {Images}},
+ url = {https://www.semanticscholar.org/paper/Learning-Multiple-Layers-of-Features-from-Tiny-Krizhevsky/5d90f06bb70a0a3dced62413346235c02b1aa086},
+ urldate = {2023-06-21},
+ abstract = {Groups at MIT and NYU have collected a dataset of millions of tiny colour images from the web. It is, in principle, an excellent dataset for unsupervised training of deep generative models, but previous researchers who have tried this have found it dicult to learn a good set of lters from the images. We show how to train a multi-layer generative model that learns to extract meaningful features which resemble those found in the human visual cortex. Using a novel parallelization algorithm to distribute the work among multiple machines connected on a network, we show how training such a model can be done in reasonable time. A second problematic aspect of the tiny images dataset is that there are no reliable class labels which makes it hard to use for object recognition experiments. We created two sets of reliable labels. The CIFAR-10 set has 6000 examples of each of 10 classes and the CIFAR-100 set has 600 examples of each of 100 non-overlapping classes. Using these labels, we show that object recognition is signicantly improved by pre-training a layer of features on a large set of unlabeled tiny images.},
+ annotation = {[TLDR] It is shown how to train a multi-layer generative model that learns to extract meaningful features which resemble those found in the human visual cortex, using a novel parallelization algorithm to distribute the work among multiple machines connected on a network.},
+ file = {Semantic Scholar Link:https\://www.semanticscholar.org/paper/Learning-Multiple-Layers-of-Features-from-Tiny-Krizhevsky/5d90f06bb70a0a3dced62413346235c02b1aa086:text/html;Full Text PDF:http\://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf:application/pdf},
+}
+
+@Misc{becker1996adult,
+ author = {Barry Becker, Ronny Kohavi},
+ date = {1996},
+ title = {Adult},
+ doi = {10.24432/C5XW20},
+ note = {Type: dataset},
+ url = {https://archive.ics.uci.edu/dataset/2},
+ urldate = {2023-06-21},
+ publisher = {UCI Machine Learning Repository},
+}
+
+@InProceedings{tolomei2017interpretable,
+ author = {Tolomei, Gabriele and Silvestri, Fabrizio and Haines, Andrew and Lalmas, Mounia},
+ booktitle = {Proceedings of the 23rd {ACM} {SIGKDD} {International} {Conference} on {Knowledge} {Discovery} and {Data} {Mining}},
+ date = {2017-08},
+ title = {Interpretable {Predictions} of {Tree}-based {Ensembles} via {Actionable} {Feature} {Tweaking}},
+ doi = {10.1145/3097983.3098039},
+ note = {arXiv:1706.06691 [stat]},
+ pages = {465--474},
+ url = {http://arxiv.org/abs/1706.06691},
+ urldate = {2023-06-21},
+ abstract = {Machine-learned models are often described as "black boxes". In many real-world applications however, models may have to sacrifice predictive power in favour of human-interpretability. When this is the case, feature engineering becomes a crucial task, which requires significant and time-consuming human effort. Whilst some features are inherently static, representing properties that cannot be influenced (e.g., the age of an individual), others capture characteristics that could be adjusted (e.g., the daily amount of carbohydrates taken). Nonetheless, once a model is learned from the data, each prediction it makes on new instances is irreversible - assuming every instance to be a static point located in the chosen feature space. There are many circumstances however where it is important to understand (i) why a model outputs a certain prediction on a given instance, (ii) which adjustable features of that instance should be modified, and finally (iii) how to alter such a prediction when the mutated instance is input back to the model. In this paper, we present a technique that exploits the internals of a tree-based ensemble classifier to offer recommendations for transforming true negative instances into positively predicted ones. We demonstrate the validity of our approach using an online advertising application. First, we design a Random Forest classifier that effectively separates between two types of ads: low (negative) and high (positive) quality ads (instances). Then, we introduce an algorithm that provides recommendations that aim to transform a low quality ad (negative instance) into a high quality one (positive instance). Finally, we evaluate our approach on a subset of the active inventory of a large ad network, Yahoo Gemini.},
+ annotation = {Comment: 10 pages, KDD 2017},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1706.06691.pdf:application/pdf},
+ keywords = {Statistics - Machine Learning, 68T01, I.2.0, I.5.1},
+}
+
+@TechReport{dandl2023counterfactuals,
+ author = {Dandl, Susanne and Hofheinz, Andreas and Binder, Martin and Bischl, Bernd and Casalicchio, Giuseppe},
+ date = {2023-04},
+ institution = {arXiv},
+ title = {counterfactuals: {An} {R} {Package} for {Counterfactual} {Explanation} {Methods}},
+ note = {arXiv:2304.06569 [cs, stat] type: article},
+ url = {http://arxiv.org/abs/2304.06569},
+ urldate = {2023-06-21},
+ abstract = {Counterfactual explanation methods provide information on how feature values of individual observations must be changed to obtain a desired prediction. Despite the increasing amount of proposed methods in research, only a few implementations exist whose interfaces and requirements vary widely. In this work, we introduce the counterfactuals R package, which provides a modular and unified R6-based interface for counterfactual explanation methods. We implemented three existing counterfactual explanation methods and propose some optional methodological extensions to generalize these methods to different scenarios and to make them more comparable. We explain the structure and workflow of the package using real use cases and show how to integrate additional counterfactual explanation methods into the package. In addition, we compared the implemented methods for a variety of models and datasets with regard to the quality of their counterfactual explanations and their runtime behavior.},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2304.06569.pdf:application/pdf},
+ keywords = {Statistics - Machine Learning, Computer Science - Machine Learning, Statistics - Computation},
+ shorttitle = {counterfactuals},
+}
+
+@TechReport{laugel2017inversea,
+ author = {Laugel, Thibault and Lesot, Marie-Jeanne and Marsala, Christophe and Renard, Xavier and Detyniecki, Marcin},
+ date = {2017-12},
+ institution = {arXiv},
+ title = {Inverse {Classification} for {Comparison}-based {Interpretability} in {Machine} {Learning}},
+ doi = {10.48550/arXiv.1712.08443},
+ note = {arXiv:1712.08443 [cs, stat] type: article},
+ url = {http://arxiv.org/abs/1712.08443},
+ urldate = {2023-06-21},
+ abstract = {In the context of post-hoc interpretability, this paper addresses the task of explaining the prediction of a classifier, considering the case where no information is available, neither on the classifier itself, nor on the processed data (neither the training nor the test data). It proposes an instance-based approach whose principle consists in determining the minimal changes needed to alter a prediction: given a data point whose classification must be explained, the proposed method consists in identifying a close neighbour classified differently, where the closeness definition integrates a sparsity constraint. This principle is implemented using observation generation in the Growing Spheres algorithm. Experimental results on two datasets illustrate the relevance of the proposed approach that can be used to gain knowledge about the classifier.},
+ annotation = {Comment: preprint},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1712.08443.pdf:application/pdf},
+ keywords = {Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
+}
+
+@TechReport{delaney2021uncertainty,
+ author = {Delaney, Eoin and Greene, Derek and Keane, Mark T.},
+ date = {2021-07},
+ institution = {arXiv},
+ title = {Uncertainty {Estimation} and {Out}-of-{Distribution} {Detection} for {Counterfactual} {Explanations}: {Pitfalls} and {Solutions}},
+ note = {arXiv:2107.09734 [cs] type: article},
+ url = {http://arxiv.org/abs/2107.09734},
+ urldate = {2023-06-23},
+ abstract = {Whilst an abundance of techniques have recently been proposed to generate counterfactual explanations for the predictions of opaque black-box systems, markedly less attention has been paid to exploring the uncertainty of these generated explanations. This becomes a critical issue in high-stakes scenarios, where uncertain and misleading explanations could have dire consequences (e.g., medical diagnosis and treatment planning). Moreover, it is often difficult to determine if the generated explanations are well grounded in the training data and sensitive to distributional shifts. This paper proposes several practical solutions that can be leveraged to solve these problems by establishing novel connections with other research works in explainability (e.g., trust scores) and uncertainty estimation (e.g., Monte Carlo Dropout). Two experiments demonstrate the utility of our proposed solutions.},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2107.09734.pdf:application/pdf},
+ keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
+ shorttitle = {Uncertainty {Estimation} and {Out}-of-{Distribution} {Detection} for {Counterfactual} {Explanations}},
+}
+
+@InProceedings{casanueva2020efficient,
+ author = {Casanueva, Iñigo and TemÄinas, Tadas and Gerz, Daniela and Henderson, Matthew and Vulić, Ivan},
+ booktitle = {Proceedings of the 2nd {Workshop} on {Natural} {Language} {Processing} for {Conversational} {AI}},
+ date = {2020-07},
+ title = {Efficient {Intent} {Detection} with {Dual} {Sentence} {Encoders}},
+ doi = {10.18653/v1/2020.nlp4convai-1.5},
+ location = {Online},
+ pages = {38--45},
+ publisher = {Association for Computational Linguistics},
+ url = {https://aclanthology.org/2020.nlp4convai-1.5},
+ urldate = {2023-06-27},
+ abstract = {Building conversational systems in new domains and with added functionality requires resource-efficient models that work under low-data regimes (i.e., in few-shot setups). Motivated by these requirements, we introduce intent detection methods backed by pretrained dual sentence encoders such as USE and ConveRT. We demonstrate the usefulness and wide applicability of the proposed intent detectors, showing that: 1) they outperform intent detectors based on fine-tuning the full BERT-Large model or using BERT as a fixed black-box encoder on three diverse intent detection data sets; 2) the gains are especially pronounced in few-shot setups (i.e., with only 10 or 30 annotated examples per intent); 3) our intent detectors can be trained in a matter of minutes on a single CPU; and 4) they are stable across different hyperparameter settings. In hope of facilitating and democratizing research focused on intention detection, we release our code, as well as a new challenging single-domain intent detection dataset comprising 13,083 annotated examples over 77 intents.},
+ file = {Full Text PDF:https\://aclanthology.org/2020.nlp4convai-1.5.pdf:application/pdf},
+}
+
+@TechReport{liu2019roberta,
+ author = {Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
+ date = {2019-07},
+ institution = {arXiv},
+ title = {{RoBERTa}: {A} {Robustly} {Optimized} {BERT} {Pretraining} {Approach}},
+ doi = {10.48550/arXiv.1907.11692},
+ note = {arXiv:1907.11692 [cs] type: article},
+ url = {http://arxiv.org/abs/1907.11692},
+ urldate = {2023-06-27},
+ abstract = {Language model pretraining has led to significant performance gains but careful comparison between different approaches is challenging. Training is computationally expensive, often done on private datasets of different sizes, and, as we will show, hyperparameter choices have significant impact on the final results. We present a replication study of BERT pretraining (Devlin et al., 2019) that carefully measures the impact of many key hyperparameters and training data size. We find that BERT was significantly undertrained, and can match or exceed the performance of every model published after it. Our best model achieves state-of-the-art results on GLUE, RACE and SQuAD. These results highlight the importance of previously overlooked design choices, and raise questions about the source of recently reported improvements. We release our models and code.},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1907.11692.pdf:application/pdf},
+ keywords = {Computer Science - Computation and Language},
+ shorttitle = {{RoBERTa}},
+}
+
+@TechReport{kunstner2020limitations,
+ author = {Kunstner, Frederik and Balles, Lukas and Hennig, Philipp},
+ date = {2020-06},
+ institution = {arXiv},
+ title = {Limitations of the {Empirical} {Fisher} {Approximation} for {Natural} {Gradient} {Descent}},
+ doi = {10.48550/arXiv.1905.12558},
+ note = {arXiv:1905.12558 [cs, stat] type: article},
+ url = {http://arxiv.org/abs/1905.12558},
+ urldate = {2023-06-30},
+ abstract = {Natural gradient descent, which preconditions a gradient descent update with the Fisher information matrix of the underlying statistical model, is a way to capture partial second-order information. Several highly visible works have advocated an approximation known as the empirical Fisher, drawing connections between approximate second-order methods and heuristics like Adam. We dispute this argument by showing that the empirical Fisher---unlike the Fisher---does not generally capture second-order information. We further argue that the conditions under which the empirical Fisher approaches the Fisher (and the Hessian) are unlikely to be met in practice, and that, even on simple optimization problems, the pathologies of the empirical Fisher can have undesirable effects.},
+ annotation = {Comment: V3: Minor corrections (typographic errors)},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1905.12558.pdf:application/pdf},
+ keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
+}
+
+@TechReport{botev2017practical,
+ author = {Botev, Aleksandar and Ritter, Hippolyt and Barber, David},
+ date = {2017-06},
+ institution = {arXiv},
+ title = {Practical {Gauss}-{Newton} {Optimisation} for {Deep} {Learning}},
+ doi = {10.48550/arXiv.1706.03662},
+ note = {arXiv:1706.03662 [stat] type: article},
+ url = {http://arxiv.org/abs/1706.03662},
+ urldate = {2023-06-30},
+ abstract = {We present an efficient block-diagonal ap- proximation to the Gauss-Newton matrix for feedforward neural networks. Our result- ing algorithm is competitive against state- of-the-art first order optimisation methods, with sometimes significant improvement in optimisation performance. Unlike first-order methods, for which hyperparameter tuning of the optimisation parameters is often a labo- rious process, our approach can provide good performance even when used with default set- tings. A side result of our work is that for piecewise linear transfer functions, the net- work objective function can have no differ- entiable local maxima, which may partially explain why such transfer functions facilitate effective optimisation.},
+ annotation = {Comment: ICML 2017},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1706.03662.pdf:application/pdf},
+ keywords = {Statistics - Machine Learning},
+}
+
+@TechReport{sharma2021sketching,
+ author = {Sharma, Apoorva and Azizan, Navid and Pavone, Marco},
+ date = {2021-02},
+ institution = {arXiv},
+ title = {Sketching {Curvature} for {Efficient} {Out}-of-{Distribution} {Detection} for {Deep} {Neural} {Networks}},
+ doi = {10.48550/arXiv.2102.12567},
+ note = {arXiv:2102.12567 [cs] type: article},
+ url = {http://arxiv.org/abs/2102.12567},
+ urldate = {2023-06-30},
+ abstract = {In order to safely deploy Deep Neural Networks (DNNs) within the perception pipelines of real-time decision making systems, there is a need for safeguards that can detect out-of-training-distribution (OoD) inputs both efficiently and accurately. Building on recent work leveraging the local curvature of DNNs to reason about epistemic uncertainty, we propose Sketching Curvature of OoD Detection (SCOD), an architecture-agnostic framework for equipping any trained DNN with a task-relevant epistemic uncertainty estimate. Offline, given a trained model and its training data, SCOD employs tools from matrix sketching to tractably compute a low-rank approximation of the Fisher information matrix, which characterizes which directions in the weight space are most influential on the predictions over the training data. Online, we estimate uncertainty by measuring how much perturbations orthogonal to these directions can alter predictions at a new test input. We apply SCOD to pre-trained networks of varying architectures on several tasks, ranging from regression to classification. We demonstrate that SCOD achieves comparable or better OoD detection performance with lower computational burden relative to existing baselines.},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2102.12567.pdf:application/pdf},
+ keywords = {Computer Science - Machine Learning},
}
-@TechReport{du2020implicit,
- author = {Du, Yilun and Mordatch, Igor},
+@TechReport{amini2019spatial,
+ author = {Amini, Alexander and Soleimany, Ava and Karaman, Sertac and Rus, Daniela},
+ date = {2019-05},
+ institution = {arXiv},
+ title = {Spatial {Uncertainty} {Sampling} for {End}-to-{End} {Control}},
+ doi = {10.48550/arXiv.1805.04829},
+ note = {arXiv:1805.04829 [cs] type: article},
+ url = {http://arxiv.org/abs/1805.04829},
+ urldate = {2023-06-30},
+ abstract = {End-to-end trained neural networks (NNs) are a compelling approach to autonomous vehicle control because of their ability to learn complex tasks without manual engineering of rule-based decisions. However, challenging road conditions, ambiguous navigation situations, and safety considerations require reliable uncertainty estimation for the eventual adoption of full-scale autonomous vehicles. Bayesian deep learning approaches provide a way to estimate uncertainty by approximating the posterior distribution of weights given a set of training data. Dropout training in deep NNs approximates Bayesian inference in a deep Gaussian process and can thus be used to estimate model uncertainty. In this paper, we propose a Bayesian NN for end-to-end control that estimates uncertainty by exploiting feature map correlation during training. This approach achieves improved model fits, as well as tighter uncertainty estimates, than traditional element-wise dropout. We evaluate our algorithms on a challenging dataset collected over many different road types, times of day, and weather conditions, and demonstrate how uncertainties can be used in conjunction with a human controller in a parallel autonomous setting.},
+ annotation = {Comment: Originally published in Neural Information Processing Systems (NIPS) Workshop on Bayesian Deep Learning 2017},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1805.04829.pdf:application/pdf},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Robotics},
+}
+
+@InProceedings{lecun1989optimal,
+ author = {LeCun, Yann and Denker, John and Solla, Sara},
+ booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
+ date = {1989},
+ title = {Optimal {Brain} {Damage}},
+ publisher = {Morgan-Kaufmann},
+ url = {https://proceedings.neurips.cc/paper/1989/hash/6c9882bbac1c7093bd25041881277658-Abstract.html},
+ urldate = {2023-06-30},
+ volume = {2},
+ abstract = {We have used information-theoretic ideas to derive a class of prac(cid:173) tical and nearly optimal schemes for adapting the size of a neural network. By removing unimportant weights from a network, sev(cid:173) eral improvements can be expected: better generalization, fewer training examples required, and improved speed of learning and/or classification. The basic idea is to use second-derivative informa(cid:173) tion to make a tradeoff between network complexity and training set error. Experiments confirm the usefulness of the methods on a real-world application.},
+ file = {Full Text PDF:https\://proceedings.neurips.cc/paper_files/paper/1989/file/6c9882bbac1c7093bd25041881277658-Paper.pdf:application/pdf},
+}
+
+@TechReport{martens2020optimizing,
+ author = {Martens, James and Grosse, Roger},
date = {2020-06},
institution = {arXiv},
- title = {Implicit {Generation} and {Generalization} in {Energy}-{Based} {Models}},
- doi = {10.48550/arXiv.1903.08689},
- note = {arXiv:1903.08689 [cs, stat] type: article},
- url = {http://arxiv.org/abs/1903.08689},
- urldate = {2023-05-16},
- abstract = {Energy based models (EBMs) are appealing due to their generality and simplicity in likelihood modeling, but have been traditionally difficult to train. We present techniques to scale MCMC based EBM training on continuous neural networks, and we show its success on the high-dimensional data domains of ImageNet32x32, ImageNet128x128, CIFAR-10, and robotic hand trajectories, achieving better samples than other likelihood models and nearing the performance of contemporary GAN approaches, while covering all modes of the data. We highlight some unique capabilities of implicit generation such as compositionality and corrupt image reconstruction and inpainting. Finally, we show that EBMs are useful models across a wide variety of tasks, achieving state-of-the-art out-of-distribution classification, adversarially robust classification, state-of-the-art continual online class learning, and coherent long term predicted trajectory rollouts.},
- file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1903.08689.pdf:application/pdf},
- keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Statistics - Machine Learning},
+ title = {Optimizing {Neural} {Networks} with {Kronecker}-factored {Approximate} {Curvature}},
+ doi = {10.48550/arXiv.1503.05671},
+ note = {arXiv:1503.05671 [cs, stat] type: article},
+ url = {http://arxiv.org/abs/1503.05671},
+ urldate = {2023-06-30},
+ abstract = {We propose an efficient method for approximating natural gradient descent in neural networks which we call Kronecker-Factored Approximate Curvature (K-FAC). K-FAC is based on an efficiently invertible approximation of a neural network's Fisher information matrix which is neither diagonal nor low-rank, and in some cases is completely non-sparse. It is derived by approximating various large blocks of the Fisher (corresponding to entire layers) as being the Kronecker product of two much smaller matrices. While only several times more expensive to compute than the plain stochastic gradient, the updates produced by K-FAC make much more progress optimizing the objective, which results in an algorithm that can be much faster than stochastic gradient descent with momentum in practice. And unlike some previously proposed approximate natural-gradient/Newton methods which use high-quality non-diagonal curvature matrices (such as Hessian-free optimization), K-FAC works very well in highly stochastic optimization regimes. This is because the cost of storing and inverting K-FAC's approximation to the curvature matrix does not depend on the amount of data used to estimate it, which is a feature typically associated only with diagonal or low-rank approximations to the curvature matrix.},
+ annotation = {Comment: Reduction ratio formula corrected. Removed incorrect claim about geodesics in footnote},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/1503.05671.pdf:application/pdf},
+ keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing, Statistics - Machine Learning},
+}
+
+@TechReport{fong2021conformal,
+ author = {Fong, Edwin and Holmes, Chris},
+ date = {2021-06},
+ institution = {arXiv},
+ title = {Conformal {Bayesian} {Computation}},
+ doi = {10.48550/arXiv.2106.06137},
+ note = {arXiv:2106.06137 [stat] type: article},
+ url = {http://arxiv.org/abs/2106.06137},
+ urldate = {2023-07-19},
+ abstract = {We develop scalable methods for producing conformal Bayesian predictive intervals with finite sample calibration guarantees. Bayesian posterior predictive distributions, \$p(y {\textbackslash}mid x)\$, characterize subjective beliefs on outcomes of interest, \$y\$, conditional on predictors, \$x\$. Bayesian prediction is well-calibrated when the model is true, but the predictive intervals may exhibit poor empirical coverage when the model is misspecified, under the so called \$\{{\textbackslash}cal\{M\}\}\$-open perspective. In contrast, conformal inference provides finite sample frequentist guarantees on predictive confidence intervals without the requirement of model fidelity. Using 'add-one-in' importance sampling, we show that conformal Bayesian predictive intervals are efficiently obtained from re-weighted posterior samples of model parameters. Our approach contrasts with existing conformal methods that require expensive refitting of models or data-splitting to achieve computational efficiency. We demonstrate the utility on a range of examples including extensions to partially exchangeable settings such as hierarchical models.},
+ annotation = {Comment: 19 pages, 4 figures, 12 tables; added references and fixed typos},
+ file = {arXiv Fulltext PDF:https\://arxiv.org/pdf/2106.06137.pdf:application/pdf},
+ keywords = {Statistics - Methodology, Statistics - Computation},
+}
+
+@Book{hyndman2018forecasting,
+ author = {Hyndman, Rob J. and Athanasopoulos, George},
+ date = {2018-05},
+ title = {Forecasting: principles and practice},
+ isbn = {9780987507112},
+ language = {en},
+ note = {Google-Books-ID: \_bBhDwAAQBAJ},
+ publisher = {OTexts},
+ abstract = {Forecasting is required in many situations. Stocking an inventory may require forecasts of demand months in advance. Telecommunication routing requires traffic forecasts a few minutes ahead. Whatever the circumstances or time horizons involved, forecasting is an important aid in effective and efficient planning.This textbook provides a comprehensive introduction to forecasting methods and presents enough information about each method for readers to use them sensibly.},
+ file = {Google Books Link:https\://books.google.com/books?id=_bBhDwAAQBAJ:text/html},
+ keywords = {Business \& Economics / Forecasting, Business \& Economics / Statistics, Computers / Databases / Data Mining, Computers / Mathematical \& Statistical Software},
+ shorttitle = {Forecasting},
+}
+
+@InProceedings{xu2021conformal,
+ author = {Xu, Chen and Xie, Yao},
+ date = {2021-07},
+ title = {Conformal prediction interval for dynamic time-series},
+ language = {en},
+ pages = {11559--11569},
+ publisher = {PMLR},
+ url = {https://proceedings.mlr.press/v139/xu21h.html},
+ urldate = {2023-07-24},
+ abstract = {We develop a method to construct distribution-free prediction intervals for dynamic time-series, called {\textbackslash}Verb{\textbar}EnbPI{\textbar} that wraps around any bootstrap ensemble estimator to construct sequential prediction intervals. {\textbackslash}Verb{\textbar}EnbPI{\textbar} is closely related to the conformal prediction (CP) framework but does not require data exchangeability. Theoretically, these intervals attain finite-sample, {\textbackslash}textit\{approximately valid\} marginal coverage for broad classes of regression functions and time-series with strongly mixing stochastic errors. Computationally, {\textbackslash}Verb{\textbar}EnbPI{\textbar} avoids overfitting and requires neither data-splitting nor training multiple ensemble estimators; it efficiently aggregates bootstrap estimators that have been trained. In general, {\textbackslash}Verb{\textbar}EnbPI{\textbar} is easy to implement, scalable to producing arbitrarily many prediction intervals sequentially, and well-suited to a wide range of regression functions. We perform extensive real-data analyses to demonstrate its effectiveness.},
+ file = {Full Text PDF:http\://proceedings.mlr.press/v139/xu21h/xu21h.pdf:application/pdf},
+ issn = {2640-3498},
+}
+
+@Book{howard2020deep,
+ author = {Howard, Jeremy and Gugger, Sylvain},
+ date = {2020-06},
+ title = {Deep {Learning} for {Coders} with fastai and {PyTorch}},
+ isbn = {9781492045496},
+ language = {en},
+ note = {Google-Books-ID: yATuDwAAQBAJ},
+ publisher = {"O'Reilly Media, Inc."},
+ abstract = {Deep learning is often viewed as the exclusive domain of math PhDs and big tech companies. But as this hands-on guide demonstrates, programmers comfortable with Python can achieve impressive results in deep learning with little math background, small amounts of data, and minimal code. How? With fastai, the first library to provide a consistent interface to the most frequently used deep learning applications.Authors Jeremy Howard and Sylvain Gugger, the creators of fastai, show you how to train a model on a wide range of tasks using fastai and PyTorch. You’ll also dive progressively further into deep learning theory to gain a complete understanding of the algorithms behind the scenes.Train models in computer vision, natural language processing, tabular data, and collaborative filteringLearn the latest deep learning techniques that matter most in practiceImprove accuracy, speed, and reliability by understanding how deep learning models workDiscover how to turn your models into web applicationsImplement deep learning algorithms from scratchConsider the ethical implications of your workGain insight from the foreword by PyTorch cofounder, Soumith Chintala},
+ file = {Google Books Link:https\://books.google.nl/books?id=yATuDwAAQBAJ:text/html},
+ keywords = {Computers / Data Science / Machine Learning, Computers / Image Processing, Computers / Computer Science, Computers / Machine Theory, Computers / Data Science / Neural Networks, Computers / Programming / General, Computers / Languages / Python, Computers / Data Science / Data Visualization},
+}
+
+@Article{samoilescu2021model,
+ author = {Samoilescu, Robert-Florian and Van Looveren, Arnaud and Klaise, Janis},
+ title = {Model-agnostic and scalable counterfactual explanations via reinforcement learning},
+ journal = {arXiv preprint arXiv:2106.02597},
+ year = {2021},
+}
+
+@Article{chen2021seven,
+ author = {Chen, Jiahao and Storchan, Victor},
+ title = {Seven challenges for harmonizing explainability requirements},
+ journal = {arXiv preprint arXiv:2108.05390},
+ year = {2021},
+}
+
+@InProceedings{wang2003multiscale,
+ author = {Wang, Zhou and Simoncelli, Eero P and Bovik, Alan C},
+ booktitle = {The Thrity-Seventh Asilomar Conference on Signals, Systems \& Computers, 2003},
+ title = {Multiscale structural similarity for image quality assessment},
+ organization = {Ieee},
+ pages = {1398--1402},
+ volume = {2},
+ year = {2003},
+}
+
+@Article{dhurandhar2018explanations,
+ author = {Dhurandhar, Amit and Chen, Pin-Yu and Luss, Ronny and Tu, Chun-Chen and Ting, Paishun and Shanmugam, Karthikeyan and Das, Payel},
+ title = {Explanations based on the missing: Towards contrastive explanations with pertinent negatives},
+ volume = {31},
+ journal = {Advances in neural information processing systems},
+ year = {2018},
+}
+
+@Article{lecun1998gradient,
+ author = {LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
+ title = {Gradient-based learning applied to document recognition},
+ number = {11},
+ pages = {2278--2324},
+ volume = {86},
+ journal = {Proceedings of the IEEE},
+ publisher = {Ieee},
+ year = {1998},
}
@Comment{jabref-meta: databaseType:biblatex;}
+
+@Comment{jabref-meta: keypatterndefault:[auth:lower][year][veryshorttitle:lower];}
+
+@Comment{jabref-meta: saveActions:disabled;
+all-text-fields[identity]
+date[normalize_date]
+month[normalize_month]
+pages[normalize_page_numbers]
+;}
diff --git a/paper/body.tex b/paper/body.tex
new file mode 100644
index 0000000000000000000000000000000000000000..e128f5f16dfb4abb45c3e573dfaa398c63475c22
--- /dev/null
+++ b/paper/body.tex
@@ -0,0 +1,260 @@
+\maketitle
+
+
+\begin{abstract}
+ Counterfactual explanations offer an intuitive and straightforward way to explain black-box models and offer algorithmic recourse to individuals. To address the need for plausible explanations, existing work has primarily relied on surrogate models to learn how the input data is distributed. This effectively reallocates the task of learning realistic explanations for the data from the model itself to the surrogate. Consequently, the generated explanations may seem plausible to humans but need not necessarily describe the behaviour of the black-box model faithfully. We formalise this notion of faithfulness through the introduction of a tailored evaluation metric and propose a novel algorithmic framework for generating \textbf{E}nergy-\textbf{C}onstrained \textbf{C}onformal \textbf{Co}unterfactuals that are only as plausible as the model permits. Through extensive empirical studies, we demonstrate that \textit{ECCCo} reconciles the need for faithfulness and plausibility. In particular, we show that for models with gradient access, it is possible to achieve state-of-the-art performance without the need for surrogate models. To do so, our framework relies solely on properties defining the black-box model itself by leveraging recent advances in energy-based modelling and conformal prediction. To our knowledge, this is the first venture in this direction for generating faithful counterfactual explanations. Thus, we anticipate that \textit{ECCCo} can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
+\end{abstract}
+
+\section{Introduction}\label{intro}
+
+Counterfactual explanations provide a powerful, flexible and intuitive way to not only explain black-box models but also offer the possibility of algorithmic recourse to affected individuals. Instead of opening the black box, counterfactual explanations work under the premise of strategically perturbing model inputs to understand model behaviour~\citep{wachter2017counterfactual}. Intuitively speaking, we generate explanations in this context by asking what-if questions of the following nature: `Our credit risk model currently predicts that this individual is not credit-worthy. What if they reduced their monthly expenditures by 10\%?'
+
+This is typically implemented by defining a target outcome $\mathbf{y}^+ \in \mathcal{Y}$ for some individual $\mathbf{x} \in \mathcal{X}=\mathbb{R}^D$ described by $D$ attributes, for which the model $M_{\theta}:\mathcal{X}\mapsto\mathcal{Y}$ initially predicts a different outcome: $M_{\theta}(\mathbf{x})\ne \mathbf{y}^+$. Counterfactuals are then searched by minimizing a loss function that compares the predicted model output to the target outcome: $\text{yloss}(M_{\theta}(\mathbf{x}),\mathbf{y}^+)$. Since counterfactual explanations work directly with the black-box model, valid counterfactuals always have full local fidelity by construction where fidelity is defined as the degree to which explanations approximate the predictions of a black-box model~\citep{molnar2022interpretable}.
+
+In situations where full fidelity is a requirement, counterfactual explanations offer a more appropriate solution to Explainable Artificial Intelligence (XAI) than other popular approaches like LIME~\citep{ribeiro2016why} and SHAP~\citep{lundberg2017unified}, which involve local surrogate models. But even full fidelity is not a sufficient condition for ensuring that an explanation \textit{faithfully} describes the behaviour of a model. That is because multiple distinct explanations can lead to the same model prediction, especially when dealing with heavily parameterized models like deep neural networks, which are underspecified by the data~\citep{wilson2020case}. In the context of counterfactuals, the idea that no two explanations are the same arises almost naturally. A key focus in the literature has therefore been to identify those explanations that are most appropriate based on a myriad of desiderata such as closeness~\citep{wachter2017counterfactual}, sparsity~\citep{schut2021generating}, actionability~\citep{ustun2019actionable} and plausibility~\citep{joshi2019realistic}.
+
+In this work, we draw closer attention to model faithfulness rather than fidelity as a desideratum for counterfactuals. We define faithfulness as the degree to which counterfactuals are consistent with what the model has learned about the data. Our key contributions are as follows: first, we show that fidelity is an insufficient evaluation metric for counterfactuals (Section~\ref{fidelity}) and propose a definition of faithfulness that gives rise to more suitable metrics (Section~\ref{faithfulness}). Next, we introduce a \textit{ECCCo}: a novel algorithmic approach aimed at generating energy-constrained conformal counterfactuals that faithfully explain model behaviour in Section~\ref{meth}. Finally, we provide extensive empirical evidence demonstrating that \textit{ECCCo} faithfully explains model behaviour and attains plausibility only when appropriate (Section~\ref{emp}).
+
+To our knowledge, this is the first venture in this direction for generating faithful counterfactuals. Thus, we anticipate that \textit{ECCCo} can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
+
+\section{Background}\label{background}
+
+While counterfactual explanations (CE) can also be generated for arbitrary regression models~\citep{spooner2021counterfactual}, existing work has primarily focused on classification problems. Let $\mathcal{Y}=(0,1)^K$ denote the one-hot-encoded output domain with $K$ classes. Then most counterfactual generators rely on gradient descent to optimize different flavours of the following counterfactual search objective:
+
+\begin{equation} \label{eq:general}
+\begin{aligned}
+\mathbf{Z}^\prime &= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \left\{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda {\text{cost}(f(\mathbf{Z}^\prime)) } \right\}
+\end{aligned}
+\end{equation}
+
+Here $\text{yloss}(\cdot)$ denotes the primary loss function, $f(\cdot)$ is a function that maps from the counterfactual state space to the feature space and $\text{cost}(\cdot)$ is either a single penalty or a collection of penalties that are used to impose constraints through regularization. Equation~\ref{eq:general} restates the baseline approach to gradient-based counterfactual search proposed by~\citet{wachter2017counterfactual} in general form as introduced by~\citet{altmeyer2023endogenous}. To explicitly account for the multiplicity of explanations, $\mathbf{Z}^\prime=\{ \mathbf{z}_l\}_L$ denotes an $L$-dimensional array of counterfactual states.
+
+The baseline approach, which we will simply refer to as \textit{Wachter}, searches a single counterfactual directly in the feature space and penalises its distance to the original factual. In this case, $f(\cdot)$ is simply the identity function and $\mathcal{Z}$ corresponds to the feature space itself. Many derivative works of~\citet{wachter2017counterfactual} have proposed new flavours of Equation~\ref{eq:general}, each of them designed to address specific \textit{desiderata} that counterfactuals ought to meet in order to properly serve both AI practitioners and individuals affected by algorithmic decision-making systems. The list of desiderata includes but is not limited to the following: sparsity, closeness~\citep{wachter2017counterfactual}, actionability~\citep{ustun2019actionable}, diversity~\citep{mothilal2020explaining}, plausibility~\citep{joshi2019realistic,poyiadzi2020face,schut2021generating}, robustness~\citep{upadhyay2021robust,pawelczyk2022probabilistically,altmeyer2023endogenous} and causality~\citep{karimi2021algorithmic}. Different counterfactual generators addressing these needs have been extensively surveyed and evaluated in various studies~\citep{verma2020counterfactual,karimi2020survey,pawelczyk2021carla,artelt2021evaluating,guidotti2022counterfactual}.
+
+The notion of plausibility is central to all of the desiderata. For example, \citet{artelt2021evaluating} find that plausibility typically also leads to improved robustness. Similarly, plausibility has also been connected to causality in the sense that plausible counterfactuals respect causal relationships~\citep{mahajan2019preserving}. Consequently, the plausibility of counterfactuals has been among the primary concerns for researchers. Achieving plausibility is equivalent to ensuring that the generated counterfactuals comply with the true and unobserved data-generating process (DGP). We define plausibility formally in this work as follows:
+
+\begin{definition}[Plausible Counterfactuals]
+ \label{def:plausible}
+ Let $\mathcal{X}|\mathbf{y}^+= p(\mathbf{x}|\mathbf{y}^+)$ denote the true conditional distribution of samples in the target class $\mathbf{y}^+$. Then for $\mathbf{x}^{\prime}$ to be considered a plausible counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}|\mathbf{y}^+$.
+\end{definition}
+
+To generate plausible counterfactuals, we first need to quantify the conditional distribution of samples in the target class ($\mathcal{X}|\mathbf{y}^+$). We can then ensure that we generate counterfactuals that comply with that distribution.
+
+One straightforward way to do this is to use surrogate models for the task. \citet{joshi2019realistic}, for example, suggest that instead of searching counterfactuals in the feature space $\mathcal{X}$, we can instead traverse a latent embedding $\mathcal{Z}$ (Equation~\ref{eq:general}) that implicitly codifies the DGP. To learn the latent embedding, they propose using a generative model such as a Variational Autoencoder (VAE). Provided the surrogate model is well-specified, their proposed approach \textit{REVISE} can yield plausible explanations. Others have proposed similar approaches: \citet{dombrowski2021diffeomorphic} traverse the base space of a normalizing flow to solve Equation~\ref{eq:general}; \citet{poyiadzi2020face} use density estimators ($\hat{p}: \mathcal{X} \mapsto [0,1]$) to constrain the counterfactuals to dense regions in the feature space; and, finally, \citet{karimi2021algorithmic} assume knowledge about the structural causal model that generates the data.
+
+A competing approach towards plausibility that is also closely related to this work instead relies on the black-box model itself. \citet{schut2021generating} show that to meet the plausibility objective we need not explicitly model the input distribution. Pointing to the undesirable engineering overhead induced by surrogate models, they propose that we rely on the implicit minimisation of predictive uncertainty instead. Their proposed methodology, which we will refer to as \textit{Schut}, solves Equation~\ref{eq:general} by greedily applying Jacobian-Based Saliency Map Attacks (JSMA) in the feature space with cross-entropy loss and no penalty at all. The authors demonstrate theoretically and empirically that their approach yields counterfactuals for which the model $M_{\theta}$ predicts the target label $\mathbf{y}^+$ with high confidence. Provided the model is well-specified, these counterfactuals are plausible. This idea hinges on the assumption that the black-box model provides well-calibrated predictive uncertainty estimates.
+
+\section{Why Fidelity is not Enough: A Motivational Example}\label{fidelity}
+
+As discussed in the introduction, any valid counterfactual also has full fidelity by construction: solutions to Equation~\ref{eq:general} are considered valid as soon as the label predicted by the model matches the target class. So while fidelity always applies, counterfactuals that address the various desiderata introduced above can look vastly different from each other.
+
+To demonstrate this with an example, we have trained a simple image classifier $M_{\theta}$ on the well-known \textit{MNIST} dataset~\citep{lecun1998mnist}: a Multi-Layer Perceptron (\textit{MLP}) with test set accuracy $> 0.9$. No measures have been taken to improve the model's adversarial robustness or its capacity for predictive uncertainty quantification. The far left panel of Figure ~\ref{fig:motiv} shows a random sample drawn from the dataset. The underlying classifier correctly predicts the label `nine' for this image. For the given factual image and model, we have used \textit{Wachter}, \textit{Schut} and \textit{REVISE} to generate one counterfactual each in the target class `seven'. The perturbed images are shown next to the factual image from left to right in Figure ~\ref{fig:motiv}. Captions on top of the images indicate the generator along with the predicted probability that the image belongs to the target class. In all cases, that probability is very high, while the counterfactuals look very different.
+
+\begin{figure}
+ \centering
+ \includegraphics[width=0.8\linewidth]{../artifacts/results/images/mnist_motivation.png}
+ \caption{Counterfactuals for turning a 9 (nine) into a 7 (seven): original image (left), then the counterfactuals generated using \textit{Wachter}, \textit{Schut} and \textit{REVISE}.}\label{fig:motiv}
+\end{figure}
+
+Since \textit{Wachter} is only concerned with closeness, the generated counterfactual is almost indistinguishable from the factual. The approach by~\citet{schut2021generating} expects a well-calibrated model that can generate predictive uncertainty estimates. Since this is not the case, the generated counterfactual looks like an adversarial example. Finally, the counterfactual generated by \textit{REVISE} looks much more plausible than the other two. But is it also more faithful to the behaviour of our \textit{MNIST} classifier? That is much less clear because the surrogate used by \textit{REVISE} introduces friction: the generated explanations no longer depend exclusively on the black-box model itself.
+
+So which of the counterfactuals most faithfully explains the behaviour of our image classifier? Fidelity cannot help us to make that judgement, because all of these counterfactuals have full fidelity. Thus, fidelity is an insufficient evaluation metric to assess the faithfulness of CE.
+
+\section{Faithful first, Plausible second}\label{faithfulness}
+
+Considering the limitations of fidelity as demonstrated in the previous section, analogous to Definition~\ref{def:plausible}, we introduce a new notion of faithfulness in the context of CE:
+
+\begin{definition}[Faithful Counterfactuals]
+ \label{def:faithful}
+ Let $\mathcal{X}_{\theta}|\mathbf{y}^+ = p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ denote the conditional distribution of $\mathbf{x}$ in the target class $\mathbf{y}^+$, where $\theta$ denotes the parameters of model $M_{\theta}$. Then for $\mathbf{x}^{\prime}$ to be considered a faithful counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}_{\theta}|\mathbf{y}^+$.
+\end{definition}
+
+In doing this, we merge in and nuance the concept of plausibility (Definition~\ref{def:plausible}) where the notion of `consistent with the data' becomes `consistent with what the model has learned about the data'.
+
+\subsection{Quantifying the Model's Generative Property}
+
+To assess counterfactuals with respect to Definition~\ref{def:faithful}, we need a way to quantify the posterior conditional distribution $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$. To this end, we draw on ideas from energy-based modelling (EBM), a subdomain of machine learning that is concerned with generative or hybrid modelling~\citep{grathwohl2020your,du2019implicit}. In particular, note that if we fix $\mathbf{y}$ to our target value $\mathbf{y}^+$, we can conditionally draw from $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ by randomly initializing $\mathbf{x}_0$ and then using Stochastic Gradient Langevin Dynamics (SGLD) as follows,
+
+\begin{equation}\label{eq:sgld}
+ \begin{aligned}
+ \mathbf{x}_{j+1} &\leftarrow \mathbf{x}_j - \frac{\epsilon_j^2}{2} \mathcal{E}_{\theta}(\mathbf{x}_j|\mathbf{y}^+) + \epsilon_j \mathbf{r}_j, && j=1,...,J
+ \end{aligned}
+\end{equation}
+
+where $\mathbf{r}_j \sim \mathcal{N}(\mathbf{0},\mathbf{I})$ is the stochastic term and the step-size $\epsilon_j$ is typically polynomially decayed~\citep{welling2011bayesian}. The term $\mathcal{E}_{\theta}(\mathbf{x}_j|\mathbf{y}^+)$ denotes the model energy conditioned on the target class label $\mathbf{y}^+$ which we specify as the negative logit corresponding to the target class label $\mathbf{y}^{+}$. To allow for faster sampling, we follow the common practice of choosing the step-size $\epsilon_j$ and the standard deviation of $\mathbf{r}_j$ separately. While $\mathbf{x}_J$ is only guaranteed to distribute as $p_{\theta}(\mathbf{x}|\mathbf{y}^{+})$ if $\epsilon \rightarrow 0$ and $J \rightarrow \infty$, the bias introduced for a small finite $\epsilon$ is negligible in practice \citep{murphy2023probabilistic}.
+
+Generating multiple samples using SGLD thus yields an empirical distribution $\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}$ that approximates what the model has learned about the input data. While in the context of EBM, this is usually done during training, we propose to repurpose this approach during inference in order to evaluate the faithfulness of model explanations. The technical appendix provides additional implementation details for any tasks related to energy-based modelling.
+
+\subsection{Quantifying the Model's Predictive Uncertainty}
+
+Faithful counterfactuals can be expected to also be plausible if the learned conditional distribution $\mathcal{X}_{\theta}|\mathbf{y}^+$ (Defintion~\ref{def:faithful}) is close to the true conditional distribution $\mathcal{X}|\mathbf{y}^+$ (Definition~\ref{def:plausible}). We can further improve the plausibility of counterfactuals without the need for surrogate models that may interfere with faithfulness by minimizing predictive uncertainty~\citep{schut2021generating}.
+Unfortunately, this idea relies on the assumption that the model itself provides predictive uncertainty estimates, which may be too restrictive in practice.
+
+To relax this assumption, we use conformal prediction (CP), an approach to predictive uncertainty quantification that has recently gained popularity~\citep{angelopoulos2021gentle,manokhin2022awesome}. Crucially for our intended application, CP is model-agnostic and can be applied during inference without placing any restrictions on model training. It works under the premise of turning heuristic notions of uncertainty into rigorous estimates by repeatedly sifting through the training data or a dedicated calibration dataset.
+
+Conformal classifiers produce prediction sets for individual inputs that include all output labels that can be reasonably attributed to the input. These sets are formed as follows,
+
+\begin{equation}\label{eq:scp}
+ \begin{aligned}
+ C_{\theta}(\mathbf{x}_i;\alpha)=\{\mathbf{y}: s(\mathbf{x}_i,\mathbf{y}) \le \hat{q}\}
+ \end{aligned}
+\end{equation}
+
+where $\hat{q}$ denotes the $(1-\alpha)$-quantile of $\mathcal{S}$ and $\alpha$ is a predetermined error rate. These sets tend to be larger for inputs that do not conform with the training data and are characterized by high predictive uncertainty. To leverage this notion of predictive uncertainty in the context of gradient-based counterfactual search, we use a smooth set size penalty introduced by~\citet{stutz2022learning}:
+
+\begin{equation}\label{eq:setsize}
+ \begin{aligned}
+ \Omega(C_{\theta}(\mathbf{x};\alpha))&=\max \left(0, \sum_{\mathbf{y}\in\mathcal{Y}}C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha) - \kappa \right)
+ \end{aligned}
+\end{equation}
+
+Here, $\kappa \in \{0,1\}$ is a hyper-parameter and $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha)$ can be interpreted as the probability of label $\mathbf{y}$ being included in the prediction set (see appendix for details). In order to compute this penalty for any black-box model, we merely need to perform a single calibration pass through a holdout set $\mathcal{D}_{\text{cal}}$. Arguably, data is typically abundant and in most applications, practitioners tend to hold out a test data set anyway. Consequently, CP removes the restriction on the family of predictive models, at the small cost of reserving a subset of the available data for calibration. This particular case of conformal prediction is referred to as \textit{split conformal prediction} (SCP) as it involves splitting the training data into a proper training dataset and a calibration dataset.
+
+\subsection{Evaluating Plausibility and Faithfulness}
+
+The parallels between our definitions of plausibility and faithfulness imply that we can also use similar evaluation metrics in both cases. Since existing work has focused heavily on plausibility, it offers a useful starting point. In particular,~\citet{guidotti2022counterfactual} have proposed an implausibility metric that measures the distance of the counterfactual from its nearest neighbour in the target class. As this distance is reduced, counterfactuals get more plausible under the assumption that the nearest neighbour itself is plausible in the sense of Definition~\ref{def:plausible}. In this work, we use the following adapted implausibility metric,
+
+\begin{equation}\label{eq:impl}
+ \begin{aligned}
+ \text{impl}(\mathbf{x}^{\prime},\mathbf{X}_{\mathbf{y}^+}) = \frac{1}{\lvert\mathbf{X}_{\mathbf{y}^+}\rvert} \sum_{\mathbf{x} \in \mathbf{X}_{\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
+ \end{aligned}
+\end{equation}
+
+where $\mathbf{x}^{\prime}$ denotes the counterfactual and $\mathbf{X}_{\mathbf{y}^+}$ is a subsample of the training data in the target class $\mathbf{y}^+$. By averaging over multiple samples in this manner, we avoid the risk that the nearest neighbour of $\mathbf{x}^{\prime}$ itself is not plausible according to Definition~\ref{def:plausible} (e.g an outlier).
+
+Equation~\ref{eq:impl} gives rise to a similar evaluation metric for unfaithfulness. We swap out the subsample of observed individuals in the target class for the set of samples generated through SGLD ($\widehat{\mathbf{X}}_{\mathbf{y}^+}$):
+
+\begin{equation}\label{eq:faith}
+ \begin{aligned}
+ \text{unfaith}(\mathbf{x}^{\prime},\widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}) = \frac{1}{\lvert \widehat{\mathbf{X}}_{\theta,\mathbf{y}^+} \rvert} \sum_{\mathbf{x} \in \widehat{\mathbf{X}}_{\theta,\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
+ \end{aligned}
+\end{equation}
+
+Our default choice for the $\text{dist}(\cdot)$ function in both cases is the Euclidean Norm. Depending on the type of input data other choices may be more adequate, which we discuss further in Section~\ref{emp:setup}.
+
+\section{Energy-Constrained Conformal Counterfactuals}\label{meth}
+
+Given our proposed notion of faithfulness, we now describe \textit{ECCCo}, our proposed framework for generating Energy-Constrained Conformal Counterfactuals. It is based on the premise that counterfactuals should first and foremost be faithful. Plausibility, as a secondary concern, is then still attainable to the degree that the black-box model itself has learned plausible explanations for the underlying data.
+
+We begin by substituting the loss function in Equation~\ref{eq:general},
+
+\begin{equation} \label{eq:eccco-start}
+ \begin{aligned}
+ \mathbf{Z}^\prime =& \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \{ {L_{\text{JEM}}(f(\mathbf{Z}^\prime);M_{\theta},\mathbf{y}^+)}+ \lambda {\text{cost}(f(\mathbf{Z}^\prime)) } \}
+ \end{aligned}
+\end{equation}
+
+where $L_{\text{JEM}}(f(\mathbf{Z}^\prime);M_{\theta},\mathbf{y}^+)$ is a hybrid loss function used in joint-energy modelling evaluated at a given counterfactual state for a given model and target outcome:
+
+\begin{equation}
+ \begin{aligned}
+ L_{\text{JEM}}(f(\mathbf{Z}^\prime); \cdot) = L_{\text{clf}}(f(\mathbf{Z}^\prime); \cdot) + L_{\text{gen}}(f(\mathbf{Z}^\prime); \cdot)
+ \end{aligned}
+\end{equation}
+
+The first term, $L_{\text{clf}}$, is any standard classification loss function such as cross-entropy loss. The second term, $L_{\text{gen}}$, is used to measure loss with respect to the generative task\footnote{In practice, regularization loss is typically also added. We follow this convention but have omitted the term here for simplicity.}. In the context of joint-energy training, $L_{\text{gen}}$ induces changes in model parameters $\theta$ that decrease the energy of observed samples and increase the energy of samples generated through SGLD~\citep{du2019implicit}.
+
+The key observation in our context is that we can rely solely on decreasing the energy of the counterfactual itself. This is sufficient to capture the generative property of the underlying model since it is implicitly captured by its parameters $\theta$. Importantly, this means that we do not need to generate conditional samples through SGLD during our counterfactual search at all as we explain in the technical appendix.
+
+This observation leads to the following simple objective function for \textit{ECCCo}:
+
+\begin{equation} \label{eq:eccco}
+ \begin{aligned}
+ \mathbf{Z}^\prime =& \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \{ {L_{\text{clf}}(f(\mathbf{Z}^\prime);M_{\theta},\mathbf{y}^+)}+ \lambda_1 {\text{cost}(f(\mathbf{Z}^\prime)) } \\
+ &+ \lambda_2 \mathcal{E}_{\theta}(f(\mathbf{Z}^\prime)|\mathbf{y}^+) + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
+ \end{aligned}
+\end{equation}
+
+The first penalty term involving $\lambda_1$ induces closeness like in~\citet{wachter2017counterfactual}. The second penalty term involving $\lambda_2$ induces faithfulness by constraining the energy of the generated counterfactual. The third and final penalty term involving $\lambda_3$ ensures that the generated counterfactual is associated with low predictive uncertainty. To tune theses hyperparameters we have relied on grid search.
+
+Concerning feature autoencoding ($f: \mathcal{Z} \mapsto \mathcal{X}$), \textit{ECCCo} does not rely on latent space search to achieve its primary objective of faithfulness. By default, we choose $f(\cdot)$ to be the identity function as in \textit{Wachter}. This is generally also enough to achieve plausibility, provided the model has learned plausible explanations for the data. In some cases, plausibility can be improved further by mapping counterfactuals to a lower-dimensional latent space. In the following, we refer to this approach as \textit{ECCCo+}: that is, \textit{ECCCo} plus dimensionality reduction.
+
+\begin{figure*}
+ \centering
+ \includegraphics[width=0.75\linewidth]{../artifacts/results/images/poc_gradient_fields.png}
+ \caption{Gradient fields and counterfactual paths for different generators. The objective is to generate a counterfactual in the blue class for a sample from the orange class. Bright yellow stars indicate conditional samples generated through SGLD. The underlying classifier is a Joint Energy Model.}\label{fig:poc}
+\end{figure*}
+
+Figure~\ref{fig:poc} illustrates how the different components in Equation~\ref{eq:eccco} affect the counterfactual search for a synthetic dataset. The underlying classifier is a Joint Energy Model (\textit{JEM}) that was trained to predict the output class (blue or orange) and generate class-conditional samples~\citep{grathwohl2020your}. We have used four different generator flavours to produce a counterfactual in the blue class for a sample from the orange class: \textit{Wachter}, which only uses the first penalty ($\lambda_2=\lambda_3=0$); \textit{ECCCo (no EBM)}, which does not constrain energy ($\lambda_2=0$); \textit{ECCCo (no CP)}, which involves no set size penalty ($\lambda_3=0$); and, finally, \textit{ECCCo}, which involves all penalties defined in Equation~\ref{eq:eccco}. Arrows indicate (negative) gradients with respect to the objective function at different points in the feature space.
+
+While \textit{Wachter} generates a valid counterfactual, it ends up close to the original starting point consistent with its objective. \textit{ECCCo (no EBM)} pushes the counterfactual further into the target domain to minimize predictive uncertainty, but the outcome is still not plausible. The counterfactual produced by \textit{ECCCo (no CP)} is energy-constrained. Since the \textit{JEM} has learned the conditional input distribution reasonably well in this case, the counterfactuals are both faithful and plausible. Finally, the outcome for \textit{ECCCo} looks similar, but the additional smooth set size penalty leads to somewhat faster convergence.
+
+\section{Empirical Analysis}\label{emp}
+
+Our goal in this section is to shed light on the following research questions:
+
+\begin{question}[Faithfulness]\label{rq:faithfulness}
+ To what extent are counterfactuals generated by \textit{ECCCo} more faithful than those produced by state-of-the-art generators?
+\end{question}
+
+\begin{question}[Balancing Desiderata]\label{rq:plausibility}
+ Compared to state-of-the-art generators, how does \textit{ECCCo} balance the two key objectives of faithfulness and plausibility?
+\end{question}
+
+The second question is motivated by the intuition that faithfulness and plausibility should coincide for models that have learned plausible explanations of the data.
+
+\subsection{Experimental Setup}\label{emp:setup}
+
+To assess and benchmark the performance of our proposed generator against the state of the art, we generate multiple counterfactuals for different models and datasets. In particular, we compare \textit{ECCCo} and its variants to the following counterfactual generators that were introduced above: firstly; \textit{Schut}, which works under the premise of minimizing predictive uncertainty; secondly, \textit{REVISE}, which is state-of-the-art (SOTA) with respect to plausibility; and, finally, \textit{Wachter}, which serves as our baseline. In the case of \textit{ECCCo+}, we use principal component analysis (PCA) for dimensionality reduction: the latent space $\mathcal{Z}$ is spanned by the first $n_z$ principal components where we choose $n_z$ to be equal to the latent dimension of the VAE used by \textit{REVISE}.
+
+For the predictive modelling tasks, we use multi-layer perceptrons (\textit{MLP}), deep ensembles, joint energy models (\textit{JEM}) and convolutional neural networks (LeNet-5 \textit{CNN}~\citep{lecun1998gradient}). Both joint-energy modelling and ensembling have been associated with improved generative properties and adversarial robustness~\citep{grathwohl2020your,lakshminarayanan2016simple}, so we expect this to be positively correlated with the plausibility of \textit{ECCCo}. To account for stochasticity, we generate multiple counterfactuals for each target class, generator, model and dataset. Full details concerning our parameter choices, training procedures and model performance can be found in the appendix.
+
+We perform benchmarks on eight datasets from different domains. From the credit and finance domain we include three tabular datasets: Give Me Some Credit (\textit{GMSC})~\citep{kaggle2011give}, \textit{German Credit}~\citet{hoffman1994german} and \textit{California Housing}~\citet{pace1997sparse}. All of these are commonly used in the related literature~\citep{karimi2020survey,altmeyer2023endogenous,pawelczyk2021carla}. Following related literature~\citep{schut2021generating,dhurandhar2018explanations} we also include two image datasets: \textit{MNIST}~\citep{lecun1998mnist} and \textit{Fashion MNIST}~\citep{xiao2017fashion}. Detailed descriptions and results for all datasets can be found in the appendix.
+
+In the following, we will focus on the most relevant results highlighted in Tables~\ref{tab:results-tabular} and~\ref{tab:results-vision}. The tables show sample averages along with standard deviations for our key evaluation metrics for the \textit{California Housing} and \textit{GMSC} datasets (Table~\ref{tab:results-tabular}) and the \textit{MNIST} dataset (Table~\ref{tab:results-vision}). For each metric, the best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). For the tabular datasets, we use the default Euclidian distance to measure unfaithfulness and implausibility as defined in Equations~\ref{eq:faith} and~\ref{eq:impl}, respectively. The third metric presented~\ref{tab:results-tabular} in Table quantifies the predictive uncertainty of the counterfactual as measured by Equation~\ref{eq:setsize}. For the vision datasets, we rely on measuring the structural dissimilarity between images for our unfaithfulness and implausibility metrics~\citep{wang2003multiscale}.
+
+\subsection{Faithfulness}
+
+\import{contents/}{table-tabular.tex}
+
+Overall, we find strong empirical evidence suggesting that \textit{ECCCo} consistently achieves state-of-the-art faithfulness. Across all models and datasets highlighted here, all variations of \textit{ECCCo} consistently outperform all other generators with respect to faithfulness, in many cases substantially. This pattern is mostly robust across all other benchmark datasets (Tables~\ref{tab:results-linearly-separable} to~\ref{tab:results-fashion-mnist} in the technical appendix).
+
+In particular, we note that the best results are generally obtained when using the full \textit{ECCCo} objective (Equation~\ref{eq:eccco}). In other words, constraining both energy and predictive uncertainty typically yields the most faithful counterfactuals. We expected the former to play a more significant role in this context and that is typically what we find across all datasets. For example, the results for \textit{GMSC} in Table~\ref{tab:results-tabular} indicate that faithfulness can be improved substantially by relying solely on the energy constraint (\textit{ECCCo (no CP)}). In some cases though, as for the \textit{California Housing} dataset, \textit{ECCCo (no EBM)} actually outperforms \textit{ECCCo (no CP)}. This indicates that predictive uncertainty minimization plays an important role in achieving faithfulness.
+
+We also generally find that the highest degree of faithfulness is obtained when the counterfactual search is performed directly in the feature space $\mathcal{X}$. While \textit{ECCCo+} typically attains high levels of faithfulness compared to most other generators, it is consistently outperformed by \textit{ECCCo}. The case is even stronger for \textit{REVISE}, which performs worst out of all generators for faithfulness on the \textit{GMSC} dataset and better only than \textit{Wachter} on \textit{California Housing}.
+
+These findings are consistent with the notion that surrogate models may inhibit faithfulness. Even though dimensionality reduction through PCA in the case of \textit{ECCCo+} can be considered a relatively mild form of intervention, the first $n_z$ principal components fail to capture some of the variation in the data, that the underlying model itself may be sensitive to. This notion is illustrated nicely in Figure~\ref{fig:mnist-bmk}, where the counterfactual produced by \textit{ECCCo} is somewhat noisier and grainier than the one produced by \textit{ECCCo+}.
+
+In conclusion, we recommend in light of the findings here to use the full \textit{ECCCo} search objective whenever model faithfulness is a key priority.
+
+\subsection{Balancing Desiderata}
+
+\begin{figure}
+ \centering
+ \includegraphics[width=1.0\linewidth]{../www/mnist_benchmark.png}
+ \caption{Counterfactuals for turning a 3 into a 5: factual (left), then the counterfactuals generated by \textit{ECCCo}, \textit{ECCCo+}, \textit{REVISE}, \textit{Schut} and \textit{Wachter}.}\label{fig:mnist-bmk}
+\end{figure}
+
+% Sample UUID of factual: UUID("1ceb06f0-5949-11ee-0e9c-dd49ccfec8c3")
+
+\import{contents/}{table-vision.tex}
+
+Overall, we find strong empirical evidence suggesting that \textit{ECCCo} can achieve near state-of-the-art plausibility without sacrificing faithfulness. Figure~\ref{fig:mnist-bmk} shows one such example taken from the \textit{MNIST} benchmark where the objective is to turn the factual three (far left) into a five. The underlying model is a LeNet-5 \textit{CNN}. The different images show the counterfactuals produced by the generators, of which all but the one produced by \textit{Schut} are valid. Both variations of \textit{ECCCo} produce plausible counterfactuals.
+
+Looking at the benchmark results presented in Tables~\ref{tab:results-tabular} and~\ref{tab:results-vision} we firstly note that although \textit{REVISE} generally performs best, \textit{ECCCo} and in particular \textit{ECCCo+} often approach SOTA performance. Upon visual inspection of the generated images we actually find that \textit{ECCCo+} performs much better than \textit{REVISE} (see appendix). Zooming in on the details we observe that \textit{ECCCo} and its variations do particularly well, whenever the underlying model has been explicitly trained to learn plausible representations of the data. For both tabular datasets in Table~\ref{tab:results-tabular}, \textit{ECCCo} improves plausibility substantially compared to the baseline. This broad pattern is mostly consistent for all other datasets, although there are notable exceptions for which \textit{ECCCo} takes the lead on both plausibility and faithfulness (see, for example, Tables~\ref{tab:results-moons} and~\ref{tab:results-german-credit} in the appendix).
+
+While we maintain that generally speaking plausibility should hinge on the quality of the model, our results also indicate that it is possible to trade off some degree of faithfulness for plausibility if needed: \textit{ECCCo+} generally outperforms other variants of \textit{ECCCo} in this context at the small cost of slightly reduced faithfulness. For the vision datasets especially, we find that \textit{ECCCo+} is consistently second only to \textit{REVISE} for all models and regularly substantially better than the baseline. Looking at the \textit{California Housing} data, latent space search markedly improves plausibility without sacrificing faithfulness: for the \textit{JEM} Ensemble, \textit{ECCCo+} performs substantially better than the baseline and only marginally worse than \textit{REVISE}. Importantly, \textit{ECCCo+} does not attain plausibility at all costs: for the MLP, plausibility is still very low but this seems to faithfully represent what the model has learned.
+
+We conclude that \textit{ECCCo} offers us a way to balance the objectives of faithfulness and plausibility. \textit{ECCCo+} can be used to tilt the scale in favour of plausibility if needed.
+
+\subsection{Additional Desiderata}
+
+While we have deliberately focused on our key metrics of interest so far, it is worth briefly considering other common desiderata for counterfactuals. With reference to the right-most columns for each dataset in Table~\ref{tab:results-tabular}, we firstly note that \textit{ECCCo} typically reduces predictive uncertainty as intended. Consistent with its design, \textit{Schut} performs well on this metric even though it does not explicitly address uncertainty as measured by conformal prediction set sizes.
+
+Another commonly discussed desideratum is closeness~\citep{wachter2017counterfactual}: counterfactuals that are closer to their factuals are associated with smaller costs to individuals in the context of algorithmic recourse. As evident from the additional tables in the appendix, the closeness desideratum tends to be negatively correlated with plausibility and faithfulness. Consequently, both \textit{REVISE} and \textit{ECCCo} generally yield more costly counterfactuals than the baseline. Nonetheless, \textit{ECCCo} does not seem to stretch costs unnecessarily: in Figure~\ref{fig:mnist-bmk} useful parts of the factual three are clearly retained.
+
+\section{Limitations}
+
+Despite having taken considerable measures to study our methodology carefully, limitations can still be identified.
+
+Firstly, we recognise that our proposed distance-based evaluation metrics for plausibility and faithfulness may not be universally applicable to all types of data. In any case, they depend on choosing a distance metric on a case-by-case basis, as we have done in this work. Arguably, commonly used metrics for measuring other desiderata such as closeness suffer from the same pitfall. We therefore think that future work on counterfactual explanations could benefit from defining universal evaluation metrics.
+
+Relatedly, we note that our proposed metric for measuring faithfulness depends on the availability of samples generated through SGLD, which in turn requires gradient access for models. This means it cannot be used to evaluate non-differentiable classifiers. Consequently, we also have not applied \textit{ECCCo} to some machine learning models commonly used for classification such as decision trees. Since \textit{ECCCo} itself does not rely on SGLD, its defining penalty functions are indeed applicable to gradient-free counterfactual generators. This is an interesting avenue for future research.
+
+Next, common challenges associated with energy-based modelling including sensitivity to scale, training instabilities and sensitivity to hyperparameters also apply to \textit{ECCCo} to some extent. In grid searches for optimal hyperparameters, we have noticed that unless properly regularized, \textit{ECCCo} is sometimes prone to overshoot for the energy constraint.
+
+Finally, while we have used ablation to understand the roles of the different components of \textit{ECCCo}, the scope of this work has prevented us from investigating the role of conformal prediction in this context more thoroughly. We have exclusively relied on split conformal prediction and have used fixed values for the predetermined error rate and other hyperparameters. Future work could benefit from more extensive ablation studies that tune hyperparameters and investigate different approaches to conformal prediction.
+
+\section{Conclusion}
+
+This work leverages ideas from energy-based modelling and conformal prediction in the context of counterfactual explanations. We have proposed a new way to generate counterfactuals that are maximally faithful to the black-box model they aim to explain. Our proposed generator, \textit{ECCCo}, produces plausible counterfactuals iff the black-box model itself has learned realistic explanations for the data, which we have demonstrated through rigorous empirical analysis. This should enable researchers and practitioners to use counterfactuals in order to discern trustworthy models from unreliable ones. While the scope of this work limits its generalizability, we believe that \textit{ECCCo} offers a solid base for future work on faithful counterfactual explanations.
\ No newline at end of file
diff --git a/paper/contents/table-california-housing-valid.tex b/paper/contents/table-california-housing-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..6ac0bde8f30fe772bfdaadf66f6309ae81e750a3
--- /dev/null
+++ b/paper/contents/table-california-housing-valid.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for California Housing dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-california-housing-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 35.00 ± 0.07** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.45 ± 0.10\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.86 ± 0.12** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.11\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.29 ± 0.13}** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.10\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 65.50 ± 1.12** & 0.74 ± 0.18*\hphantom{*} & 0.16 ± 0.03\hphantom{*}\hphantom{*} & 1.53 ± 0.67\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 84.14 ± 0.28** & 0.71 ± 0.25*\hphantom{*} & 0.17 ± 0.04\hphantom{*}\hphantom{*} & 2.88 ± 1.12\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 83.73 ± 0.29** & 0.74 ± 0.17*\hphantom{*} & 0.17 ± 0.04\hphantom{*}\hphantom{*} & 1.50 ± 0.66\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 71.82 ± 0.18** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.10\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 94.71 ± 1.61** & \textbf{0.68 ± 0.31}*\hphantom{*} & 0.29 ± 0.10\hphantom{*}\hphantom{*} & 3.09 ± 1.67\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 84.61 ± 0.44** & 1.03 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.00}** & 0.74 ± 0.60\hphantom{*}\hphantom{*} & \textbf{0.79 ± 0.14}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 110.44 ± 0.42\hphantom{*}\hphantom{*} & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.44 ± 0.10}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.24 ± 0.37** & 1.06 ± 0.52\hphantom{*}\hphantom{*} & 0.47 ± 0.27\hphantom{*}\hphantom{*} & 0.73 ± 0.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.76 ± 0.83** & 1.12 ± 0.48\hphantom{*}\hphantom{*} & 0.68 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.15}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.26 ± 0.63}** & 1.07 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.28\hphantom{*}\hphantom{*} & 0.71 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 65.20 ± 7.42** & 0.73 ± 0.26*\hphantom{*} & \textbf{0.11 ± 0.02}** & 1.67 ± 0.87\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 81.01 ± 3.33** & 0.63 ± 0.20** & 0.12 ± 0.02** & 2.92 ± 1.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 84.79 ± 0.49** & 0.73 ± 0.27*\hphantom{*} & 0.12 ± 0.03** & 1.61 ± 0.88\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 77.03 ± 5.57** & 1.07 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.28\hphantom{*}\hphantom{*} & 0.71 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 94.52 ± 0.61** & \textbf{0.54 ± 0.12}** & 0.21 ± 0.12** & 2.91 ± 1.29\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 85.81 ± 4.65** & 1.21 ± 0.39\hphantom{*}\hphantom{*} & 0.73 ± 0.16\hphantom{*}\hphantom{*} & 0.93 ± 0.34\hphantom{*}\hphantom{*} & \textbf{0.83 ± 0.09}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 107.85 ± 2.52\hphantom{*}\hphantom{*} & 1.12 ± 0.48\hphantom{*}\hphantom{*} & 0.68 ± 0.22\hphantom{*}\hphantom{*} & 0.62 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.19 ± 0.09** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 0.15 ± 0.04\hphantom{*}\hphantom{*} & 0.88 ± 0.77\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 42.06 ± 0.12** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 0.16 ± 0.04\hphantom{*}\hphantom{*} & 0.85 ± 0.75\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.52 ± 0.16}** & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.86 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 67.91 ± 1.63** & 3.41 ± 2.28\hphantom{*}\hphantom{*} & 0.14 ± 0.02\hphantom{*}\hphantom{*} & 5.11 ± 3.02\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 82.72 ± 1.12** & 2.71 ± 2.32\hphantom{*}\hphantom{*} & 0.17 ± 0.12\hphantom{*}\hphantom{*} & 4.75 ± 3.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 88.72 ± 2.28** & 3.40 ± 2.28\hphantom{*}\hphantom{*} & 0.14 ± 0.03\hphantom{*}\hphantom{*} & 5.09 ± 3.04\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 75.47 ± 1.60** & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.86 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 98.98 ± 0.23** & \textbf{0.64 ± 0.19}*\hphantom{*} & 0.21 ± 0.15\hphantom{*}\hphantom{*} & 2.55 ± 0.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 87.66 ± 2.05** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.13 ± 0.00}** & 1.11 ± 0.68\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 114.38 ± 2.14\hphantom{*}\hphantom{*} & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.16 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.84 ± 0.74}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.18 ± 0.15** & 3.13 ± 9.43\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 1.05 ± 0.69\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.94 ± 0.20** & 3.15 ± 9.43\hphantom{*}\hphantom{*} & 0.17 ± 0.12\hphantom{*}\hphantom{*} & 1.05 ± 0.68\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.39 ± 0.38}** & 3.13 ± 9.43\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 1.04 ± 0.70\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 66.75 ± 2.37** & 4.51 ± 9.21\hphantom{*}\hphantom{*} & 0.13 ± 0.11\hphantom{*}\hphantom{*} & 3.86 ± 2.37\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 81.44 ± 1.53** & 4.42 ± 9.07\hphantom{*}\hphantom{*} & 0.19 ± 0.22\hphantom{*}\hphantom{*} & 4.85 ± 2.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 88.08 ± 0.71** & 4.51 ± 9.21\hphantom{*}\hphantom{*} & 0.13 ± 0.11\hphantom{*}\hphantom{*} & 3.83 ± 2.38\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 76.41 ± 0.99** & 3.13 ± 9.43\hphantom{*}\hphantom{*} & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 1.04 ± 0.70\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 97.37 ± 1.05** & \textbf{0.80 ± 0.29}** & 0.16 ± 0.06\hphantom{*}\hphantom{*} & 2.78 ± 1.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 86.84 ± 1.62** & 3.13 ± 9.22\hphantom{*}\hphantom{*} & \textbf{0.13 ± 0.09}\hphantom{*}\hphantom{*} & 1.68 ± 1.38\hphantom{*}\hphantom{*} & \textbf{0.59 ± 0.26}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 112.38 ± 1.77\hphantom{*}\hphantom{*} & 3.13 ± 9.43\hphantom{*}\hphantom{*} & 0.17 ± 0.12\hphantom{*}\hphantom{*} & \textbf{1.02 ± 0.68}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-california-housing.tex b/paper/contents/table-california-housing.tex
new file mode 100644
index 0000000000000000000000000000000000000000..06fb87e626379f6638a9e45533663dfafe18a64b
--- /dev/null
+++ b/paper/contents/table-california-housing.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for California Housing dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-california-housing} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 35.00 ± 0.07** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.45 ± 0.10\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.86 ± 0.12** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.11\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.29 ± 0.13}** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.10\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 65.50 ± 1.12** & 0.74 ± 0.18*\hphantom{*} & 0.16 ± 0.03\hphantom{*}\hphantom{*} & 1.53 ± 0.67\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 84.16 ± 0.29** & 0.74 ± 0.30\hphantom{*}\hphantom{*} & 0.16 ± 0.05\hphantom{*}\hphantom{*} & 2.92 ± 1.11\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 83.73 ± 0.29** & 0.74 ± 0.17*\hphantom{*} & 0.17 ± 0.04\hphantom{*}\hphantom{*} & 1.50 ± 0.66\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 71.82 ± 0.18** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.10\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 94.71 ± 1.61** & \textbf{0.68 ± 0.31}*\hphantom{*} & 0.29 ± 0.10\hphantom{*}\hphantom{*} & 3.09 ± 1.67\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 84.61 ± 0.44** & 1.03 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.00}** & 0.74 ± 0.60\hphantom{*}\hphantom{*} & \textbf{0.79 ± 0.14}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 110.44 ± 0.42\hphantom{*}\hphantom{*} & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 0.12 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.44 ± 0.10}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.24 ± 0.37** & 1.06 ± 0.52\hphantom{*}\hphantom{*} & 0.47 ± 0.27\hphantom{*}\hphantom{*} & 0.73 ± 0.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.76 ± 0.83** & 1.12 ± 0.48\hphantom{*}\hphantom{*} & 0.68 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.15}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.26 ± 0.63}** & 1.07 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.28\hphantom{*}\hphantom{*} & 0.71 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 65.20 ± 7.42** & 0.73 ± 0.26*\hphantom{*} & \textbf{0.11 ± 0.02}** & 1.67 ± 0.87\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 81.01 ± 3.33** & 0.63 ± 0.20** & 0.12 ± 0.02** & 2.92 ± 1.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 84.79 ± 0.49** & 0.73 ± 0.27*\hphantom{*} & 0.12 ± 0.03** & 1.61 ± 0.88\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 77.03 ± 5.57** & 1.07 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.28\hphantom{*}\hphantom{*} & 0.71 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 94.52 ± 0.61** & \textbf{0.54 ± 0.12}** & 0.21 ± 0.12** & 2.91 ± 1.29\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 85.81 ± 4.65** & 1.21 ± 0.39\hphantom{*}\hphantom{*} & 0.73 ± 0.16\hphantom{*}\hphantom{*} & 0.93 ± 0.34\hphantom{*}\hphantom{*} & \textbf{0.83 ± 0.09}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 107.85 ± 2.52\hphantom{*}\hphantom{*} & 1.12 ± 0.48\hphantom{*}\hphantom{*} & 0.68 ± 0.22\hphantom{*}\hphantom{*} & 0.62 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.19 ± 0.09** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 0.15 ± 0.04\hphantom{*}\hphantom{*} & 0.88 ± 0.77\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 42.06 ± 0.12** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 0.16 ± 0.04\hphantom{*}\hphantom{*} & 0.85 ± 0.75\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.52 ± 0.16}** & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.86 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 67.91 ± 1.63** & 3.41 ± 2.28\hphantom{*}\hphantom{*} & 0.14 ± 0.02\hphantom{*}\hphantom{*} & 5.11 ± 3.02\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 82.72 ± 1.12** & 2.71 ± 2.32\hphantom{*}\hphantom{*} & 0.17 ± 0.12\hphantom{*}\hphantom{*} & 4.75 ± 3.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 88.72 ± 2.28** & 3.40 ± 2.28\hphantom{*}\hphantom{*} & 0.14 ± 0.03\hphantom{*}\hphantom{*} & 5.09 ± 3.04\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 75.47 ± 1.60** & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.86 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 98.98 ± 0.23** & \textbf{0.64 ± 0.19}*\hphantom{*} & 0.21 ± 0.15\hphantom{*}\hphantom{*} & 2.55 ± 0.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 87.66 ± 2.05** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.13 ± 0.00}** & 1.11 ± 0.68\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 114.38 ± 2.14\hphantom{*}\hphantom{*} & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 0.16 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.84 ± 0.74}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 35.18 ± 0.15** & 3.09 ± 9.23\hphantom{*}\hphantom{*} & 0.15 ± 0.12\hphantom{*}\hphantom{*} & 1.20 ± 0.99\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 41.95 ± 0.20** & 3.11 ± 9.23\hphantom{*}\hphantom{*} & 0.16 ± 0.12\hphantom{*}\hphantom{*} & 1.19 ± 0.98\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{30.38 ± 0.38}** & 3.09 ± 9.23\hphantom{*}\hphantom{*} & 0.15 ± 0.11\hphantom{*}\hphantom{*} & 1.18 ± 0.99\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 66.76 ± 2.33** & 4.41 ± 9.03\hphantom{*}\hphantom{*} & 0.13 ± 0.11\hphantom{*}\hphantom{*} & 3.85 ± 2.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 81.44 ± 1.53** & 4.42 ± 9.07\hphantom{*}\hphantom{*} & 0.19 ± 0.22\hphantom{*}\hphantom{*} & 4.85 ± 2.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo (no CP) & 88.12 ± 0.73** & 4.41 ± 9.03\hphantom{*}\hphantom{*} & 0.13 ± 0.11\hphantom{*}\hphantom{*} & 3.82 ± 2.33\hphantom{*}\hphantom{*} & 0.01 ± 0.03\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 76.38 ± 0.99** & 3.09 ± 9.23\hphantom{*}\hphantom{*} & 0.15 ± 0.11\hphantom{*}\hphantom{*} & 1.18 ± 0.99\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 97.27 ± 1.13** & \textbf{1.01 ± 1.07}*\hphantom{*} & 0.16 ± 0.07\hphantom{*}\hphantom{*} & 4.72 ± 9.79\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & Schut & 86.84 ± 1.62** & 3.13 ± 9.22\hphantom{*}\hphantom{*} & \textbf{0.13 ± 0.09}\hphantom{*}\hphantom{*} & 1.68 ± 1.38\hphantom{*}\hphantom{*} & \textbf{0.59 ± 0.26}** & \textbf{1.00 ± 0.00}**\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 112.34 ± 1.75\hphantom{*}\hphantom{*} & 3.09 ± 9.23\hphantom{*}\hphantom{*} & 0.16 ± 0.12\hphantom{*}\hphantom{*} & \textbf{1.16 ± 0.97}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-circles-valid.tex b/paper/contents/table-circles-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..4a1a7bafdfc7c5e1945b5cf6c5a8885a5211ab20
--- /dev/null
+++ b/paper/contents/table-circles-valid.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Circles dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-circles-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.19 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.18 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.17 ± 0.20*\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.42 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.10 ± 0.18\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.50 ± 0.57\hphantom{*}\hphantom{*} & 0.48 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.10 ± 0.18}\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.36 ± 0.46\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & \textbf{0.03 ± 0.01}** & \textbf{0.43 ± 0.01}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.29 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.49 ± 0.67\hphantom{*}\hphantom{*} & 0.75 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.70 ± 0.55\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.41 ± 0.50\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.21 ± 0.24*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.21 ± 0.23*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.20 ± 0.23*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.52 ± 0.61\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.50 ± 0.58\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.51 ± 0.57\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & \textbf{0.06 ± 0.01}** & \textbf{0.43 ± 0.01}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.28 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.67 ± 0.91\hphantom{*}\hphantom{*} & 1.14 ± 0.96\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.30 ± 1.32\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.07}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.52 ± 0.59\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.25 ± 0.43}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.15 ± 0.18*\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.39 ± 0.46\hphantom{*}\hphantom{*} & 0.36 ± 0.06*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.91 ± 0.17}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.40 ± 0.46\hphantom{*}\hphantom{*} & \textbf{0.36 ± 0.06}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.91 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.37 ± 0.45\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & \textbf{0.02 ± 0.01}** & 0.43 ± 0.01\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.38 ± 0.57\hphantom{*}\hphantom{*} & 0.58 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.41 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.40 ± 0.49\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.16 ± 0.19*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.39 ± 0.47\hphantom{*}\hphantom{*} & \textbf{0.39 ± 0.07}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.97 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.42 ± 0.50\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.97 ± 0.15}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.39 ± 0.47\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & \textbf{0.05 ± 0.01}** & 0.43 ± 0.01*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.39 ± 0.59\hphantom{*}\hphantom{*} & 0.62 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.48 ± 0.32\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.07}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.43 ± 0.52\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-circles.tex b/paper/contents/table-circles.tex
new file mode 100644
index 0000000000000000000000000000000000000000..713afffbec8ed759bdb20aa036cf751b8c8f5524
--- /dev/null
+++ b/paper/contents/table-circles.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Circles dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-circles} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.19 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.18 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.17 ± 0.20}*\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.42 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.10 ± 0.18\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.50 ± 0.57\hphantom{*}\hphantom{*} & \textbf{0.48 ± 0.20}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.10 ± 0.18\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.36 ± 0.46\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.30 ± 0.36\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.05 ± 0.34}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.63 ± 0.49\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.49 ± 0.67\hphantom{*}\hphantom{*} & 0.75 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.70 ± 0.55\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.41 ± 0.50\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.21 ± 0.24*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.21 ± 0.23*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.20 ± 0.23}*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.52 ± 0.61\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.50 ± 0.58\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.51 ± 0.57\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.31 ± 0.34\hphantom{*}\hphantom{*} & \textbf{0.60 ± 0.23}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.04 ± 0.34}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.63 ± 0.49\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.67 ± 0.91\hphantom{*}\hphantom{*} & 1.14 ± 0.96\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.30 ± 1.32\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.07}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.52 ± 0.59\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.15 ± 0.18}*\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.39 ± 0.46\hphantom{*}\hphantom{*} & 0.36 ± 0.06*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.91 ± 0.17}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.40 ± 0.46\hphantom{*}\hphantom{*} & \textbf{0.36 ± 0.06}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.91 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.37 ± 0.45\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.29 ± 0.36\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.04 ± 0.33\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.63 ± 0.49\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.38 ± 0.57\hphantom{*}\hphantom{*} & 0.58 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.41 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.40 ± 0.49\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.16 ± 0.19}*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.39 ± 0.47\hphantom{*}\hphantom{*} & \textbf{0.39 ± 0.07}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.97 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.42 ± 0.50\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.97 ± 0.15}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.39 ± 0.47\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.31 ± 0.35\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.03 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.63 ± 0.49\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.39 ± 0.59\hphantom{*}\hphantom{*} & 0.62 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.48 ± 0.32\hphantom{*}\hphantom{*} & \textbf{0.01 ± 0.07}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.43 ± 0.52\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-fashion-mnist-valid.tex b/paper/contents/table-fashion-mnist-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..779c47803485e7d76218f4ad5dd5afbeb6441700
--- /dev/null
+++ b/paper/contents/table-fashion-mnist-valid.tex
@@ -0,0 +1,71 @@
+\begin{table}
+
+\caption{All results for Fashion MNIST dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-fashion-mnist-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.04\hphantom{*}\hphantom{*} & \textbf{4.47 ± 0.04}** & 139.56 ± 15.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.44 ± 0.05\hphantom{*}\hphantom{*} & 4.51 ± 0.12\hphantom{*}\hphantom{*} & 108.30 ± 22.79\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.05\hphantom{*}\hphantom{*} & 4.61 ± 0.16\hphantom{*}\hphantom{*} & 210.72 ± 49.52\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.07\hphantom{*}\hphantom{*} & 4.80 ± 0.18\hphantom{*}\hphantom{*} & 234.08 ± 84.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.34 ± 0.08}\hphantom{*}\hphantom{*} & 4.61 ± 0.25\hphantom{*}\hphantom{*} & \textbf{8.94 ± 1.61}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 4.58 ± 0.09\hphantom{*}\hphantom{*} & 50.51 ± 26.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.41 ± 0.05\hphantom{*}\hphantom{*} & 2.80 ± 0.33\hphantom{*}\hphantom{*} & 172.74 ± 34.98\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.42 ± 0.05\hphantom{*}\hphantom{*} & 2.85 ± 0.34\hphantom{*}\hphantom{*} & 136.61 ± 33.78\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.23 ± 0.01\hphantom{*}\hphantom{*} & 0.38 ± 0.05\hphantom{*}\hphantom{*} & 2.81 ± 0.32\hphantom{*}\hphantom{*} & 234.00 ± 50.03\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.23 ± 0.01\hphantom{*}\hphantom{*} & 0.33 ± 0.06*\hphantom{*} & 2.94 ± 0.31\hphantom{*}\hphantom{*} & 246.19 ± 80.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.23 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.27 ± 0.08}*\hphantom{*} & \textbf{2.78 ± 0.35}\hphantom{*}\hphantom{*} & \textbf{9.93 ± 0.21}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 3.04 ± 0.26\hphantom{*}\hphantom{*} & 96.21 ± 45.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.04\hphantom{*}\hphantom{*} & 2.74 ± 0.08*\hphantom{*} & 156.09 ± 23.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.42 ± 0.05\hphantom{*}\hphantom{*} & \textbf{2.67 ± 0.05}** & 113.87 ± 29.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.38 ± 0.06\hphantom{*}\hphantom{*} & 2.81 ± 0.22\hphantom{*}\hphantom{*} & 191.29 ± 39.55\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.25 ± 0.00\hphantom{*}\hphantom{*} & 0.34 ± 0.06*\hphantom{*} & 2.99 ± 0.38\hphantom{*}\hphantom{*} & 212.99 ± 65.33\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.26 ± 0.06}** & 3.03 ± 0.26\hphantom{*}\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.97 ± 0.01}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash LeNet-5} & Wachter & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 2.88 ± 0.11\hphantom{*}\hphantom{*} & 70.95 ± 31.30\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.42 ± 0.05\hphantom{*}\hphantom{*} & 2.88 ± 0.04** & 165.55 ± 24.75\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.23 ± 0.01*\hphantom{*} & 0.42 ± 0.05\hphantom{*}\hphantom{*} & \textbf{2.85 ± 0.03}** & 150.61 ± 29.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.38 ± 0.06\hphantom{*}\hphantom{*} & 2.95 ± 0.20\hphantom{*}\hphantom{*} & 216.29 ± 42.99\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.06*\hphantom{*} & 3.20 ± 0.21\hphantom{*}\hphantom{*} & 237.81 ± 83.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.05}** & 3.18 ± 0.33\hphantom{*}\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.07\hphantom{*}\hphantom{*} & 3.02 ± 0.08\hphantom{*}\hphantom{*} & 79.26 ± 35.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.42 ± 0.04\hphantom{*}\hphantom{*} & 1.96 ± 0.10** & 183.09 ± 35.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.23 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.05\hphantom{*}\hphantom{*} & \textbf{1.91 ± 0.09}** & 149.03 ± 40.89\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.37 ± 0.05\hphantom{*}\hphantom{*} & 2.03 ± 0.22*\hphantom{*} & 220.16 ± 44.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.05*\hphantom{*} & 2.30 ± 0.20\hphantom{*}\hphantom{*} & 237.20 ± 78.25\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.33 ± 0.05}*\hphantom{*} & 2.32 ± 0.16\hphantom{*}\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.06\hphantom{*}\hphantom{*} & 2.26 ± 0.15\hphantom{*}\hphantom{*} & 99.89 ± 46.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-fashion-mnist.tex b/paper/contents/table-fashion-mnist.tex
new file mode 100644
index 0000000000000000000000000000000000000000..e37bea512f21a05391462082ac25036a2202d756
--- /dev/null
+++ b/paper/contents/table-fashion-mnist.tex
@@ -0,0 +1,71 @@
+\begin{table}
+
+\caption{All results for Fashion MNIST dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-fashion-mnist} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.04\hphantom{*}\hphantom{*} & 4.47 ± 0.04** & 139.56 ± 15.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.44 ± 0.04\hphantom{*}\hphantom{*} & 4.52 ± 0.12\hphantom{*}\hphantom{*} & 110.71 ± 22.80\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.05\hphantom{*}\hphantom{*} & 4.23 ± 1.35\hphantom{*}\hphantom{*} & 215.86 ± 50.52\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.80 ± 0.40\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.35 ± 0.07}\hphantom{*}\hphantom{*} & 4.76 ± 0.51\hphantom{*}\hphantom{*} & 239.17 ± 87.89\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.93 ± 0.26\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & \textbf{0.91 ± 1.90}*\hphantom{*} & \textbf{9.86 ± 0.67}** & \textbf{0.99 ± 0.00}** & 0.13 ± 0.34\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 4.58 ± 0.09\hphantom{*}\hphantom{*} & 50.51 ± 26.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.41 ± 0.05\hphantom{*}\hphantom{*} & 2.77 ± 0.43\hphantom{*}\hphantom{*} & 174.05 ± 37.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.99 ± 0.10}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.43 ± 0.04\hphantom{*}\hphantom{*} & 2.86 ± 0.43\hphantom{*}\hphantom{*} & 143.55 ± 37.34\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.80 ± 0.40\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.38 ± 0.05\hphantom{*}\hphantom{*} & 2.74 ± 0.58\hphantom{*}\hphantom{*} & 236.42 ± 50.53\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.23 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.33 ± 0.06}*\hphantom{*} & 2.88 ± 0.52\hphantom{*}\hphantom{*} & 248.33 ± 82.25\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & \textbf{0.22 ± 0.76}** & \textbf{9.99 ± 0.06}** & \textbf{0.99 ± 0.00}** & 0.08 ± 0.27\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 3.02 ± 0.40\hphantom{*}\hphantom{*} & 99.13 ± 49.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.04\hphantom{*}\hphantom{*} & 2.69 ± 0.39\hphantom{*}\hphantom{*} & 157.17 ± 24.41\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.98 ± 0.14}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.42 ± 0.05\hphantom{*}\hphantom{*} & 2.61 ± 0.38\hphantom{*}\hphantom{*} & 115.10 ± 29.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.97 ± 0.17\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.06\hphantom{*}\hphantom{*} & 2.47 ± 0.94\hphantom{*}\hphantom{*} & 193.83 ± 37.56\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.83 ± 0.38\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.25 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.35 ± 0.07}\hphantom{*}\hphantom{*} & 2.60 ± 1.07\hphantom{*}\hphantom{*} & 220.75 ± 67.70\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.86 ± 0.35\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.00\hphantom{*}\hphantom{*} & 0.40 ± 0.06\hphantom{*}\hphantom{*} & \textbf{0.22 ± 0.80}** & \textbf{9.99 ± 0.08}** & \textbf{0.97 ± 0.01}** & 0.05 ± 0.22\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash LeNet-5} & Wachter & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 2.83 ± 0.43\hphantom{*}\hphantom{*} & 76.69 ± 35.18\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.86 ± 0.35\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.42 ± 0.05\hphantom{*}\hphantom{*} & 2.88 ± 0.04** & 165.55 ± 24.75\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.23 ± 0.01*\hphantom{*} & 0.42 ± 0.05\hphantom{*}\hphantom{*} & 2.85 ± 0.03** & 150.61 ± 29.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.38 ± 0.06\hphantom{*}\hphantom{*} & 2.72 ± 0.83\hphantom{*}\hphantom{*} & 220.40 ± 43.04\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.89 ± 0.31\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.35 ± 0.07}*\hphantom{*} & 3.10 ± 0.59\hphantom{*}\hphantom{*} & 240.92 ± 84.84\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.07\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.97}** & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 0.04 ± 0.20\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.42 ± 0.07\hphantom{*}\hphantom{*} & 3.03 ± 0.09\hphantom{*}\hphantom{*} & 83.18 ± 37.53\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.93 ± 0.26\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.08 ± 0.01}** & 0.42 ± 0.04\hphantom{*}\hphantom{*} & 1.96 ± 0.10** & 183.09 ± 35.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.23 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.05\hphantom{*}\hphantom{*} & 1.91 ± 0.09** & 149.03 ± 40.89\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.37 ± 0.05\hphantom{*}\hphantom{*} & 1.96 ± 0.47\hphantom{*}\hphantom{*} & 223.36 ± 46.62\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.94 ± 0.24\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.34 ± 0.06}*\hphantom{*} & 2.18 ± 0.54\hphantom{*}\hphantom{*} & 239.44 ± 78.65\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.22\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.06\hphantom{*}\hphantom{*} & \textbf{0.09 ± 0.46}** & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 0.04 ± 0.20\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 2.27 ± 0.17\hphantom{*}\hphantom{*} & 103.81 ± 49.66\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-german-credit-valid.tex b/paper/contents/table-german-credit-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..b30ba573a4dc115d6cf1fc4927834fb80b4e33cb
--- /dev/null
+++ b/paper/contents/table-german-credit-valid.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for German Credit dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-german-credit-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 2.26 ± 0.14** & 4.97 ± 0.77\hphantom{*}\hphantom{*} & 0.58 ± 0.16\hphantom{*}\hphantom{*} & 1.54 ± 2.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.43 ± 0.37** & 4.98 ± 0.78\hphantom{*}\hphantom{*} & 0.58 ± 0.16\hphantom{*}\hphantom{*} & 1.46 ± 2.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.24 ± 0.41}** & 4.98 ± 0.78\hphantom{*}\hphantom{*} & 0.58 ± 0.16\hphantom{*}\hphantom{*} & 1.51 ± 2.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 6.11 ± 2.61\hphantom{*}\hphantom{*} & 4.56 ± 0.51\hphantom{*}\hphantom{*} & 0.54 ± 0.11\hphantom{*}\hphantom{*} & 4.12 ± 3.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 6.12 ± 2.12\hphantom{*}\hphantom{*} & \textbf{3.57 ± 0.39}** & 0.51 ± 0.01** & 12.09 ± 2.70\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 5.40 ± 3.00\hphantom{*}\hphantom{*} & 4.55 ± 0.50\hphantom{*}\hphantom{*} & 0.56 ± 0.14\hphantom{*}\hphantom{*} & 4.37 ± 3.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 5.29 ± 2.18\hphantom{*}\hphantom{*} & 4.98 ± 0.78\hphantom{*}\hphantom{*} & 0.58 ± 0.16\hphantom{*}\hphantom{*} & 1.51 ± 2.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 5.56 ± 2.24\hphantom{*}\hphantom{*} & 3.83 ± 0.60*\hphantom{*} & 0.52 ± 0.02** & 11.87 ± 2.64\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 6.83 ± 2.76\hphantom{*}\hphantom{*} & 4.89 ± 0.73\hphantom{*}\hphantom{*} & \textbf{0.50 ± 0.00}** & \textbf{0.90 ± 0.82}\hphantom{*}\hphantom{*} & \textbf{0.89 ± 0.07}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 6.30 ± 1.43\hphantom{*}\hphantom{*} & 4.98 ± 0.78\hphantom{*}\hphantom{*} & 0.58 ± 0.16\hphantom{*}\hphantom{*} & 1.50 ± 2.10\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.19 ± 0.26** & 4.71 ± 0.69\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 3.99 ± 3.60\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.38 ± 0.38** & 4.70 ± 0.72\hphantom{*}\hphantom{*} & 0.93 ± 0.06\hphantom{*}\hphantom{*} & 3.92 ± 3.59\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.07 ± 0.38}** & 4.73 ± 0.71\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 3.96 ± 3.57\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 4.51 ± 4.12\hphantom{*}\hphantom{*} & 4.08 ± 0.57*\hphantom{*} & 0.79 ± 0.22\hphantom{*}\hphantom{*} & 10.11 ± 4.93\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 4.44 ± 4.30\hphantom{*}\hphantom{*} & 3.71 ± 0.45** & 0.81 ± 0.20\hphantom{*}\hphantom{*} & 14.05 ± 3.51\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 3.95 ± 3.99\hphantom{*}\hphantom{*} & 4.08 ± 0.57*\hphantom{*} & 0.80 ± 0.21\hphantom{*}\hphantom{*} & 10.10 ± 4.94\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 4.55 ± 2.76\hphantom{*}\hphantom{*} & 4.73 ± 0.71\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 3.96 ± 3.57\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 4.02 ± 3.09\hphantom{*}\hphantom{*} & \textbf{3.61 ± 0.38}** & 0.91 ± 0.07\hphantom{*}\hphantom{*} & 12.70 ± 3.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 6.78 ± 3.86\hphantom{*}\hphantom{*} & 4.68 ± 0.74\hphantom{*}\hphantom{*} & \textbf{0.77 ± 0.16}*\hphantom{*} & \textbf{3.70 ± 1.12}\hphantom{*}\hphantom{*} & \textbf{0.72 ± 0.13}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 5.43 ± 3.58\hphantom{*}\hphantom{*} & 4.73 ± 0.72\hphantom{*}\hphantom{*} & 0.93 ± 0.05\hphantom{*}\hphantom{*} & 3.89 ± 3.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.38 ± 0.34** & 4.81 ± 0.64\hphantom{*}\hphantom{*} & 0.72 ± 0.10\hphantom{*}\hphantom{*} & 4.74 ± 3.06\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.57 ± 0.49** & 4.83 ± 0.62\hphantom{*}\hphantom{*} & 0.78 ± 0.10\hphantom{*}\hphantom{*} & 4.54 ± 3.03\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.38 ± 0.35}** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.73 ± 0.11\hphantom{*}\hphantom{*} & 4.70 ± 3.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 6.58 ± 1.69\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.75 ± 0.09\hphantom{*}\hphantom{*} & 4.67 ± 3.10\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 8.09 ± 1.31\hphantom{*}\hphantom{*} & 3.84 ± 0.53*\hphantom{*} & 0.67 ± 0.15\hphantom{*}\hphantom{*} & 13.67 ± 2.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 6.82 ± 1.56\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.78 ± 0.08\hphantom{*}\hphantom{*} & 4.55 ± 3.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 6.65 ± 1.87\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.73 ± 0.11\hphantom{*}\hphantom{*} & 4.70 ± 3.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 7.00 ± 0.40\hphantom{*}\hphantom{*} & \textbf{3.66 ± 0.29}** & 0.66 ± 0.19\hphantom{*}\hphantom{*} & 13.59 ± 2.64\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 8.28 ± 1.44\hphantom{*}\hphantom{*} & 4.96 ± 0.74\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.14}*\hphantom{*} & \textbf{4.13 ± 0.91}\hphantom{*}\hphantom{*} & \textbf{0.66 ± 0.14}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 6.58 ± 2.00\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.77 ± 0.10\hphantom{*}\hphantom{*} & 4.56 ± 3.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.29 ± 0.46** & 4.62 ± 0.53\hphantom{*}\hphantom{*} & 0.90 ± 0.10\hphantom{*}\hphantom{*} & 6.24 ± 3.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.42 ± 0.43** & 4.64 ± 0.56\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 6.14 ± 3.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.20 ± 0.37}** & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 6.17 ± 3.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 5.71 ± 3.12\hphantom{*}\hphantom{*} & 4.67 ± 0.56\hphantom{*}\hphantom{*} & 0.91 ± 0.07\hphantom{*}\hphantom{*} & 6.31 ± 3.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 9.54 ± 0.11\hphantom{*}\hphantom{*} & \textbf{3.35 ± 0.23}** & 0.62 ± 0.03** & 14.60 ± 0.73\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 5.46 ± 2.62\hphantom{*}\hphantom{*} & 4.67 ± 0.56\hphantom{*}\hphantom{*} & 0.92 ± 0.06\hphantom{*}\hphantom{*} & 6.27 ± 3.33\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 5.99 ± 2.84\hphantom{*}\hphantom{*} & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 6.17 ± 3.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 7.90 ± 0.38\hphantom{*}\hphantom{*} & 3.46 ± 0.18** & \textbf{0.51 ± 0.01}** & 14.14 ± 1.02\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 9.26 ± 2.00\hphantom{*}\hphantom{*} & 4.93 ± 0.63\hphantom{*}\hphantom{*} & 0.68 ± 0.12** & \textbf{4.66 ± 0.80}*\hphantom{*} & \textbf{0.74 ± 0.07}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 6.38 ± 2.33\hphantom{*}\hphantom{*} & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.92 ± 0.07\hphantom{*}\hphantom{*} & 6.09 ± 3.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-german-credit.tex b/paper/contents/table-german-credit.tex
new file mode 100644
index 0000000000000000000000000000000000000000..e1a20a61d1c4148e9606c9522fb3e94512c1ea71
--- /dev/null
+++ b/paper/contents/table-german-credit.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for German Credit dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-german-credit} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 2.26 ± 0.13** & 4.98 ± 0.76\hphantom{*}\hphantom{*} & 0.56 ± 0.19\hphantom{*}\hphantom{*} & 1.79 ± 2.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.42 ± 0.37** & 5.00 ± 0.76\hphantom{*}\hphantom{*} & 0.56 ± 0.19\hphantom{*}\hphantom{*} & 1.71 ± 2.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.22 ± 0.42}** & 5.00 ± 0.77\hphantom{*}\hphantom{*} & 0.55 ± 0.19\hphantom{*}\hphantom{*} & 1.75 ± 2.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 5.64 ± 2.62\hphantom{*}\hphantom{*} & 4.53 ± 0.49\hphantom{*}\hphantom{*} & 0.46 ± 0.23\hphantom{*}\hphantom{*} & 5.16 ± 3.82\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.84 ± 0.37\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 5.30 ± 2.12\hphantom{*}\hphantom{*} & \textbf{3.70 ± 0.38}** & \textbf{0.35 ± 0.24}\hphantom{*}\hphantom{*} & 13.17 ± 2.96\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.68 ± 0.48\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 5.05 ± 2.96\hphantom{*}\hphantom{*} & 4.53 ± 0.49\hphantom{*}\hphantom{*} & 0.50 ± 0.23\hphantom{*}\hphantom{*} & 5.19 ± 3.80\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 5.20 ± 2.19\hphantom{*}\hphantom{*} & 5.00 ± 0.77\hphantom{*}\hphantom{*} & 0.55 ± 0.19\hphantom{*}\hphantom{*} & 1.75 ± 2.40\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 5.56 ± 2.24\hphantom{*}\hphantom{*} & 3.83 ± 0.60*\hphantom{*} & 0.52 ± 0.02*\hphantom{*} & 11.87 ± 2.64\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & Schut & 6.43 ± 2.81\hphantom{*}\hphantom{*} & 4.95 ± 0.71\hphantom{*}\hphantom{*} & 0.44 ± 0.17\hphantom{*}\hphantom{*} & \textbf{1.40 ± 1.56}\hphantom{*}\hphantom{*} & \textbf{0.86 ± 0.12}** & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 6.23 ± 1.45\hphantom{*}\hphantom{*} & 5.00 ± 0.76\hphantom{*}\hphantom{*} & 0.56 ± 0.19\hphantom{*}\hphantom{*} & 1.74 ± 2.39\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.19 ± 0.26** & 4.71 ± 0.69\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 3.99 ± 3.60\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.38 ± 0.38** & 4.70 ± 0.72\hphantom{*}\hphantom{*} & 0.93 ± 0.06\hphantom{*}\hphantom{*} & 3.92 ± 3.59\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.07 ± 0.38}** & 4.73 ± 0.71\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 3.96 ± 3.57\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 4.51 ± 4.12\hphantom{*}\hphantom{*} & 4.08 ± 0.57*\hphantom{*} & 0.79 ± 0.22\hphantom{*}\hphantom{*} & 10.11 ± 4.93\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 4.44 ± 4.30\hphantom{*}\hphantom{*} & 3.71 ± 0.45** & 0.81 ± 0.20\hphantom{*}\hphantom{*} & 14.05 ± 3.51\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 3.95 ± 3.99\hphantom{*}\hphantom{*} & 4.08 ± 0.57*\hphantom{*} & 0.80 ± 0.21\hphantom{*}\hphantom{*} & 10.10 ± 4.94\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 4.55 ± 2.76\hphantom{*}\hphantom{*} & 4.73 ± 0.71\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 3.96 ± 3.57\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 4.02 ± 3.09\hphantom{*}\hphantom{*} & \textbf{3.61 ± 0.38}** & 0.91 ± 0.07\hphantom{*}\hphantom{*} & 12.70 ± 3.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 5.78 ± 3.82\hphantom{*}\hphantom{*} & 4.94 ± 0.87\hphantom{*}\hphantom{*} & \textbf{0.58 ± 0.36}\hphantom{*}\hphantom{*} & 4.01 ± 1.12\hphantom{*}\hphantom{*} & \textbf{0.71 ± 0.12}** & 0.76 ± 0.44\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 5.43 ± 3.58\hphantom{*}\hphantom{*} & 4.73 ± 0.72\hphantom{*}\hphantom{*} & 0.93 ± 0.05\hphantom{*}\hphantom{*} & \textbf{3.89 ± 3.58}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.38 ± 0.34** & 4.81 ± 0.64\hphantom{*}\hphantom{*} & 0.72 ± 0.10\hphantom{*}\hphantom{*} & 4.74 ± 3.06\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.57 ± 0.49** & 4.83 ± 0.62\hphantom{*}\hphantom{*} & 0.78 ± 0.10\hphantom{*}\hphantom{*} & 4.54 ± 3.03\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.38 ± 0.35}** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.73 ± 0.11\hphantom{*}\hphantom{*} & 4.70 ± 3.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 6.58 ± 1.69\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.75 ± 0.09\hphantom{*}\hphantom{*} & 4.67 ± 3.10\hphantom{*}\hphantom{*} & 0.00 ± 0.01\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 6.80 ± 1.65\hphantom{*}\hphantom{*} & 3.79 ± 0.39** & \textbf{0.35 ± 0.36}*\hphantom{*} & 13.45 ± 2.69\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.52 ± 0.51\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 6.82 ± 1.56\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.78 ± 0.08\hphantom{*}\hphantom{*} & 4.55 ± 3.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 6.65 ± 1.87\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.73 ± 0.11\hphantom{*}\hphantom{*} & 4.70 ± 3.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 6.78 ± 0.40\hphantom{*}\hphantom{*} & \textbf{3.70 ± 0.23}** & 0.37 ± 0.36*\hphantom{*} & 13.52 ± 2.79\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.56 ± 0.51\hphantom{*}\hphantom{*}\\
+
+ & Schut & 7.86 ± 1.41\hphantom{*}\hphantom{*} & 4.92 ± 0.71\hphantom{*}\hphantom{*} & 0.44 ± 0.30*\hphantom{*} & \textbf{4.37 ± 0.86}\hphantom{*}\hphantom{*} & \textbf{0.64 ± 0.13}** & 0.72 ± 0.46\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 6.58 ± 2.00\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 0.77 ± 0.10\hphantom{*}\hphantom{*} & 4.56 ± 3.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 2.29 ± 0.46** & 4.62 ± 0.53\hphantom{*}\hphantom{*} & 0.90 ± 0.10\hphantom{*}\hphantom{*} & 6.24 ± 3.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 2.42 ± 0.43** & 4.64 ± 0.56\hphantom{*}\hphantom{*} & 0.91 ± 0.09\hphantom{*}\hphantom{*} & 6.14 ± 3.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{2.20 ± 0.37}** & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 6.17 ± 3.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 5.71 ± 3.12\hphantom{*}\hphantom{*} & 4.67 ± 0.56\hphantom{*}\hphantom{*} & 0.91 ± 0.07\hphantom{*}\hphantom{*} & 6.31 ± 3.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 6.12 ± 3.09\hphantom{*}\hphantom{*} & \textbf{3.50 ± 0.20}** & 0.27 ± 0.31** & 14.04 ± 2.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.51\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 5.46 ± 2.62\hphantom{*}\hphantom{*} & 4.67 ± 0.56\hphantom{*}\hphantom{*} & 0.92 ± 0.06\hphantom{*}\hphantom{*} & 6.27 ± 3.33\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 5.99 ± 2.84\hphantom{*}\hphantom{*} & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.90 ± 0.09\hphantom{*}\hphantom{*} & 6.17 ± 3.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 6.29 ± 1.47\hphantom{*}\hphantom{*} & 3.55 ± 0.14** & \textbf{0.23 ± 0.26}** & 13.76 ± 2.45\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.44 ± 0.51\hphantom{*}\hphantom{*}\\
+
+ & Schut & 6.20 ± 3.31\hphantom{*}\hphantom{*} & 4.71 ± 0.59\hphantom{*}\hphantom{*} & 0.33 ± 0.36*\hphantom{*} & \textbf{4.84 ± 0.57}** & \textbf{0.63 ± 0.14}** & 0.48 ± 0.51\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 6.38 ± 2.33\hphantom{*}\hphantom{*} & 4.66 ± 0.57\hphantom{*}\hphantom{*} & 0.92 ± 0.07\hphantom{*}\hphantom{*} & 6.09 ± 3.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-gmsc-valid.tex b/paper/contents/table-gmsc-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..0cc8cb071c1947c724ddcbe89cd026e7747e2db6
--- /dev/null
+++ b/paper/contents/table-gmsc-valid.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for GMSC dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-gmsc-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 43.87 ± 0.12** & 0.90 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.02\hphantom{*}\hphantom{*} & 0.74 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.64 ± 0.15** & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{34.01 ± 0.15}** & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 75.05 ± 3.44** & 0.81 ± 0.28\hphantom{*}\hphantom{*} & 0.22 ± 0.08\hphantom{*}\hphantom{*} & 0.91 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 97.57 ± 0.19\hphantom{*}\hphantom{*} & 0.75 ± 0.15\hphantom{*}\hphantom{*} & 0.26 ± 0.09\hphantom{*}\hphantom{*} & 2.77 ± 0.91\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 79.72 ± 0.57** & 0.82 ± 0.26\hphantom{*}\hphantom{*} & 0.26 ± 0.16\hphantom{*}\hphantom{*} & 0.86 ± 0.36\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 90.34 ± 0.35\hphantom{*}\hphantom{*} & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 117.40 ± 1.20\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.09}** & 0.27 ± 0.12\hphantom{*}\hphantom{*} & 2.41 ± 0.54\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 113.59 ± 0.49\hphantom{*}\hphantom{*} & 1.09 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.14 ± 0.02}*\hphantom{*} & 1.48 ± 0.69\hphantom{*}\hphantom{*} & \textbf{0.63 ± 0.17}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 83.19 ± 0.37\hphantom{*}\hphantom{*} & 0.88 ± 0.26\hphantom{*}\hphantom{*} & 0.16 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.17}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 42.73 ± 2.77** & 0.88 ± 0.20\hphantom{*}\hphantom{*} & 0.12 ± 0.01\hphantom{*}\hphantom{*} & 0.79 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.26 ± 0.83** & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.45 ± 0.48}** & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 76.91 ± 4.04*\hphantom{*} & 0.75 ± 0.18\hphantom{*}\hphantom{*} & 0.27 ± 0.19\hphantom{*}\hphantom{*} & 0.82 ± 0.30\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 96.86 ± 7.49\hphantom{*}\hphantom{*} & 0.68 ± 0.13*\hphantom{*} & 0.23 ± 0.19\hphantom{*}\hphantom{*} & 2.53 ± 0.88\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 81.35 ± 0.82*\hphantom{*} & 0.79 ± 0.19\hphantom{*}\hphantom{*} & 0.25 ± 0.21\hphantom{*}\hphantom{*} & 0.83 ± 0.37\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 87.72 ± 1.01\hphantom{*}\hphantom{*} & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 111.55 ± 5.38\hphantom{*}\hphantom{*} & \textbf{0.66 ± 0.11}*\hphantom{*} & 0.23 ± 0.08\hphantom{*}\hphantom{*} & 2.29 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 110.80 ± 5.99\hphantom{*}\hphantom{*} & 1.25 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.10 ± 0.01}** & 1.81 ± 0.96\hphantom{*}\hphantom{*} & \textbf{0.72 ± 0.15}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 82.68 ± 3.58\hphantom{*}\hphantom{*} & 0.86 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.73 ± 0.21}\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 43.76 ± 0.16** & 1.01 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.03\hphantom{*}\hphantom{*} & 0.73 ± 0.26\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.79 ± 0.19** & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.69 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.79 ± 0.23}** & 1.00 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 0.71 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 80.15 ± 1.86** & 2.03 ± 1.30\hphantom{*}\hphantom{*} & 0.18 ± 0.03*\hphantom{*} & 3.20 ± 2.35\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 98.25 ± 0.57\hphantom{*}\hphantom{*} & 1.64 ± 1.01\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 4.63 ± 2.90\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 82.52 ± 1.18*\hphantom{*} & 2.02 ± 1.30\hphantom{*}\hphantom{*} & 0.18 ± 0.03\hphantom{*}\hphantom{*} & 3.18 ± 2.36\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 92.86 ± 1.05\hphantom{*}\hphantom{*} & 1.00 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 0.71 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 118.36 ± 1.68\hphantom{*}\hphantom{*} & \textbf{0.71 ± 0.37}\hphantom{*}\hphantom{*} & 0.28 ± 0.08\hphantom{*}\hphantom{*} & 2.50 ± 1.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 114.37 ± 1.21\hphantom{*}\hphantom{*} & 1.32 ± 0.72\hphantom{*}\hphantom{*} & \textbf{0.16 ± 0.00}** & 1.55 ± 0.87\hphantom{*}\hphantom{*} & \textbf{0.65 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 84.37 ± 0.99\hphantom{*}\hphantom{*} & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 0.21 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.67 ± 0.22}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 43.75 ± 0.24** & 0.88 ± 0.24\hphantom{*}\hphantom{*} & 0.20 ± 0.03\hphantom{*}\hphantom{*} & 0.77 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.73 ± 0.18** & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.74 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.80 ± 0.20}** & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.72 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 80.01 ± 2.52*\hphantom{*} & 2.28 ± 1.43\hphantom{*}\hphantom{*} & 0.18 ± 0.03*\hphantom{*} & 4.03 ± 3.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 97.39 ± 0.27\hphantom{*}\hphantom{*} & 1.99 ± 1.36\hphantom{*}\hphantom{*} & 0.19 ± 0.04\hphantom{*}\hphantom{*} & 5.97 ± 3.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 82.81 ± 2.34\hphantom{*}\hphantom{*} & 2.26 ± 1.44\hphantom{*}\hphantom{*} & 0.18 ± 0.03\hphantom{*}\hphantom{*} & 4.00 ± 3.19\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 92.61 ± 2.18\hphantom{*}\hphantom{*} & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.72 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 117.95 ± 2.03\hphantom{*}\hphantom{*} & \textbf{0.64 ± 0.09}** & 0.28 ± 0.09\hphantom{*}\hphantom{*} & 2.36 ± 0.65\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 113.56 ± 0.44\hphantom{*}\hphantom{*} & 1.28 ± 0.39\hphantom{*}\hphantom{*} & \textbf{0.16 ± 0.01}** & 1.82 ± 1.22\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 83.89 ± 2.61\hphantom{*}\hphantom{*} & 0.86 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.25}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-gmsc.tex b/paper/contents/table-gmsc.tex
new file mode 100644
index 0000000000000000000000000000000000000000..f355d89196315e631ea3037789acbfd1f661f702
--- /dev/null
+++ b/paper/contents/table-gmsc.tex
@@ -0,0 +1,91 @@
+\begin{table}
+
+\caption{All results for GMSC dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-gmsc} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 43.87 ± 0.12** & 0.90 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.02\hphantom{*}\hphantom{*} & 0.74 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.64 ± 0.15** & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.19\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{34.01 ± 0.15}** & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 74.71 ± 3.49** & 0.82 ± 0.27\hphantom{*}\hphantom{*} & 0.20 ± 0.10\hphantom{*}\hphantom{*} & 0.88 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.28\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 97.53 ± 0.20\hphantom{*}\hphantom{*} & 0.72 ± 0.15*\hphantom{*} & 0.22 ± 0.13\hphantom{*}\hphantom{*} & 2.76 ± 0.89\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.84 ± 0.37\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 79.70 ± 0.57** & 0.81 ± 0.26\hphantom{*}\hphantom{*} & 0.25 ± 0.16\hphantom{*}\hphantom{*} & 0.85 ± 0.35\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.96 ± 0.20\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 90.34 ± 0.35\hphantom{*}\hphantom{*} & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 0.15 ± 0.03\hphantom{*}\hphantom{*} & 0.72 ± 0.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 117.40 ± 1.20\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.09}** & 0.27 ± 0.12\hphantom{*}\hphantom{*} & 2.41 ± 0.54\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 113.59 ± 0.49\hphantom{*}\hphantom{*} & 1.09 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.14 ± 0.02}*\hphantom{*} & 1.48 ± 0.69\hphantom{*}\hphantom{*} & \textbf{0.63 ± 0.17}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM} & Wachter & 83.19 ± 0.37\hphantom{*}\hphantom{*} & 0.88 ± 0.26\hphantom{*}\hphantom{*} & 0.16 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.17}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 42.73 ± 2.77** & 0.88 ± 0.20\hphantom{*}\hphantom{*} & 0.12 ± 0.01\hphantom{*}\hphantom{*} & 0.79 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.26 ± 0.83** & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.45 ± 0.48}** & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 76.30 ± 4.15*\hphantom{*} & 0.78 ± 0.20\hphantom{*}\hphantom{*} & 0.24 ± 0.20\hphantom{*}\hphantom{*} & 0.80 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 96.72 ± 7.19\hphantom{*}\hphantom{*} & 0.68 ± 0.12*\hphantom{*} & 0.21 ± 0.19\hphantom{*}\hphantom{*} & 2.56 ± 0.85\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.28\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 82.18 ± 4.59\hphantom{*}\hphantom{*} & 0.78 ± 0.19\hphantom{*}\hphantom{*} & 0.22 ± 0.22\hphantom{*}\hphantom{*} & 0.83 ± 0.35\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 87.72 ± 1.01\hphantom{*}\hphantom{*} & 0.87 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & 0.76 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 111.55 ± 5.38\hphantom{*}\hphantom{*} & \textbf{0.66 ± 0.11}*\hphantom{*} & 0.23 ± 0.08\hphantom{*}\hphantom{*} & 2.29 ± 0.76\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 110.80 ± 5.99\hphantom{*}\hphantom{*} & 1.25 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.10 ± 0.01}** & 1.81 ± 0.96\hphantom{*}\hphantom{*} & \textbf{0.72 ± 0.15}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 82.68 ± 3.58\hphantom{*}\hphantom{*} & 0.86 ± 0.20\hphantom{*}\hphantom{*} & 0.13 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.73 ± 0.21}\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 43.76 ± 0.16** & 1.01 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.03\hphantom{*}\hphantom{*} & 0.73 ± 0.26\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.79 ± 0.19** & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.69 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.79 ± 0.23}** & 1.00 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 0.71 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 80.15 ± 1.86** & 2.03 ± 1.30\hphantom{*}\hphantom{*} & 0.18 ± 0.03*\hphantom{*} & 3.20 ± 2.35\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 98.25 ± 0.57\hphantom{*}\hphantom{*} & 1.64 ± 1.01\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 4.63 ± 2.90\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 82.52 ± 1.18*\hphantom{*} & 2.02 ± 1.30\hphantom{*}\hphantom{*} & 0.18 ± 0.03\hphantom{*}\hphantom{*} & 3.18 ± 2.36\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 92.86 ± 1.05\hphantom{*}\hphantom{*} & 1.00 ± 0.77\hphantom{*}\hphantom{*} & 0.20 ± 0.04\hphantom{*}\hphantom{*} & 0.71 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 118.36 ± 1.68\hphantom{*}\hphantom{*} & \textbf{0.71 ± 0.37}\hphantom{*}\hphantom{*} & 0.28 ± 0.08\hphantom{*}\hphantom{*} & 2.50 ± 1.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 114.37 ± 1.21\hphantom{*}\hphantom{*} & 1.32 ± 0.72\hphantom{*}\hphantom{*} & \textbf{0.16 ± 0.00}** & 1.55 ± 0.87\hphantom{*}\hphantom{*} & \textbf{0.65 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP} & Wachter & 84.37 ± 0.99\hphantom{*}\hphantom{*} & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 0.21 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.67 ± 0.22}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 43.75 ± 0.24** & 0.88 ± 0.24\hphantom{*}\hphantom{*} & 0.20 ± 0.03\hphantom{*}\hphantom{*} & 0.77 ± 0.32\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 39.73 ± 0.18** & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.74 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{33.80 ± 0.20}** & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.72 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 80.01 ± 2.52*\hphantom{*} & 2.28 ± 1.43\hphantom{*}\hphantom{*} & 0.18 ± 0.03*\hphantom{*} & 4.03 ± 3.17\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 97.39 ± 0.27\hphantom{*}\hphantom{*} & 1.99 ± 1.36\hphantom{*}\hphantom{*} & 0.19 ± 0.04\hphantom{*}\hphantom{*} & 5.97 ± 3.44\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 82.81 ± 2.34\hphantom{*}\hphantom{*} & 2.26 ± 1.44\hphantom{*}\hphantom{*} & 0.18 ± 0.03\hphantom{*}\hphantom{*} & 4.00 ± 3.19\hphantom{*}\hphantom{*} & 0.00 ± 0.02\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 92.61 ± 2.18\hphantom{*}\hphantom{*} & 0.87 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & 0.72 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 117.95 ± 2.03\hphantom{*}\hphantom{*} & \textbf{0.64 ± 0.09}** & 0.28 ± 0.09\hphantom{*}\hphantom{*} & 2.36 ± 0.65\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 113.56 ± 0.44\hphantom{*}\hphantom{*} & 1.28 ± 0.39\hphantom{*}\hphantom{*} & \textbf{0.16 ± 0.01}** & 1.82 ± 1.22\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.19}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-10}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 83.89 ± 2.61\hphantom{*}\hphantom{*} & 0.86 ± 0.24\hphantom{*}\hphantom{*} & 0.21 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.69 ± 0.25}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-linearly-separable-valid.tex b/paper/contents/table-linearly-separable-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..e415cb93da3b5cb752036a320c2209adc67edf30
--- /dev/null
+++ b/paper/contents/table-linearly-separable-valid.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Linearly Separable dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-linearly-separable-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.03 ± 0.01** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & \textbf{0.03 ± 0.01}** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.04 ± 0.02** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.07 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.05\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.07 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.08 ± 0.05}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.08 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.10 ± 0.00\hphantom{*}\hphantom{*} & 0.17 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.45 ± 0.14}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.16\hphantom{*}\hphantom{*} & 0.37 ± 0.30\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.99 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.24 ± 0.25}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.09 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.03 ± 0.01}** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.09 ± 0.03\hphantom{*}\hphantom{*} & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.26 ± 0.00\hphantom{*}\hphantom{*} & 0.18 ± 0.01\hphantom{*}\hphantom{*} & 0.74 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.41 ± 0.17}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.33 ± 0.06\hphantom{*}\hphantom{*} & 0.66 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.50 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & \textbf{0.13 ± 0.01}** & \textbf{0.09 ± 0.03}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.38 ± 0.03\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.98 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.43 ± 0.14}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.36 ± 0.04\hphantom{*}\hphantom{*} & 0.28 ± 0.10\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.71 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.35 ± 0.23}*\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.12 ± 0.01** & \textbf{0.08 ± 0.02}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.12 ± 0.01** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.12 ± 0.02** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & \textbf{0.08 ± 0.04}** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.08 ± 0.04** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.37 ± 0.03\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.98 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.41 ± 0.13}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.37 ± 0.06\hphantom{*}\hphantom{*} & 0.30 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.70 ± 0.12\hphantom{*}\hphantom{*} & \textbf{0.50 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-linearly-separable.tex b/paper/contents/table-linearly-separable.tex
new file mode 100644
index 0000000000000000000000000000000000000000..875b7afe2876d8123b1d1bda9cc87450c129e183
--- /dev/null
+++ b/paper/contents/table-linearly-separable.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Linearly Separable dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-linearly-separable} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.03 ± 0.01** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & \textbf{0.03 ± 0.01}** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.04 ± 0.02** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.07 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.05\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.07 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.08 ± 0.05}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.08 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.15 ± 0.04\hphantom{*}\hphantom{*} & 0.17 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.15}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.52 ± 0.50\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.16\hphantom{*}\hphantom{*} & 0.37 ± 0.30\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.99 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.24 ± 0.25}\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.09 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.03 ± 0.01}** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.09 ± 0.03\hphantom{*}\hphantom{*} & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.27 ± 0.01\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.36 ± 0.37\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.15}** & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.48 ± 0.50\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.33 ± 0.06\hphantom{*}\hphantom{*} & 0.66 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.50 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & \textbf{0.13 ± 0.01}** & \textbf{0.09 ± 0.03}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.40 ± 0.03\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.54 ± 0.49\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.13}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.55 ± 0.50\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.36 ± 0.04\hphantom{*}\hphantom{*} & 0.28 ± 0.10\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.71 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.35 ± 0.23}*\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.12 ± 0.01** & \textbf{0.08 ± 0.02}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.12 ± 0.01** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.12 ± 0.02** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & \textbf{0.08 ± 0.04}** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.08 ± 0.04** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.40 ± 0.04\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.52 ± 0.49\hphantom{*}\hphantom{*} & \textbf{0.40 ± 0.13}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.53 ± 0.50\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.37 ± 0.06\hphantom{*}\hphantom{*} & 0.30 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.70 ± 0.12\hphantom{*}\hphantom{*} & \textbf{0.50 ± 0.00}** & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-main.tex b/paper/contents/table-main.tex
index 056b1e285591553f74a04592b817d1627f9d0b68..309401bc781584de97cd0558a2581b6378b03e62 100644
--- a/paper/contents/table-main.tex
+++ b/paper/contents/table-main.tex
@@ -1,53 +1,51 @@
-\begin{table}
+\begin{table*}[t]
\caption{Results for datasets from different domains: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-main} \newline}
\centering
\resizebox{\linewidth}{!}{
-\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X}
+\begin{tabular}[t]{llcccccc}
\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{GMSC} & \multicolumn{2}{c}{California Housing} & \multicolumn{2}{c}{German Credit} \\
+\multicolumn{2}{c}{ } & \multicolumn{2}{c}{Linearly Separable} & \multicolumn{2}{c}{GMSC} & \multicolumn{2}{c}{MNIST} \\
\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓\\
\midrule
- & ECCCo-L1 & 0.90 ± 0.26\hphantom{*}\hphantom{*} & 43.87 ± 0.12** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 35.00 ± 0.07** & 4.98 ± 0.76\hphantom{*}\hphantom{*} & 2.26 ± 0.13**\\
+ & ECCCo & 0.25 ± 0.07*\hphantom{*} & 0.07 ± 0.03\hphantom{*}\hphantom{*} & \textbf{80.15 ± 1.86}** & 2.03 ± 1.30\hphantom{*}\hphantom{*} & \textbf{0.22 ± 0.01}** & 0.42 ± 0.02\hphantom{*}\hphantom{*}\\
- & ECCCo-L1 (no CP) & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 39.64 ± 0.15** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 41.86 ± 0.12** & 5.00 ± 0.76\hphantom{*}\hphantom{*} & 2.42 ± 0.37**\\
+ & ECCCo+ & & & 98.25 ± 0.57\hphantom{*}\hphantom{*} & 1.64 ± 1.01\hphantom{*}\hphantom{*} & 0.23 ± 0.01*\hphantom{*} & 0.32 ± 0.02*\hphantom{*}\\
- & ECCCo-L1 (no EBM) & 0.89 ± 0.26\hphantom{*}\hphantom{*} & \textbf{34.01 ± 0.15}** & 1.02 ± 0.31\hphantom{*}\hphantom{*} & \textbf{30.29 ± 0.13}** & 5.00 ± 0.77\hphantom{*}\hphantom{*} & \textbf{2.22 ± 0.42}**\\
+ & ECCCo (no CP) & \textbf{0.25 ± 0.07}*\hphantom{*} & \textbf{0.07 ± 0.03}\hphantom{*}\hphantom{*} & 82.52 ± 1.18*\hphantom{*} & 2.02 ± 1.30\hphantom{*}\hphantom{*} & & \\
- & ECCCo & 0.82 ± 0.27\hphantom{*}\hphantom{*} & 74.71 ± 3.49** & 0.74 ± 0.18*\hphantom{*} & 65.50 ± 1.12** & 4.53 ± 0.49\hphantom{*}\hphantom{*} & 5.64 ± 2.62\hphantom{*}\hphantom{*}\\
+ & ECCCo (no EBM) & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & 92.86 ± 1.05\hphantom{*}\hphantom{*} & 1.00 ± 0.77\hphantom{*}\hphantom{*} & & \\
- & ECCCo+ & 0.72 ± 0.15*\hphantom{*} & 97.53 ± 0.20\hphantom{*}\hphantom{*} & 0.74 ± 0.30\hphantom{*}\hphantom{*} & 84.16 ± 0.29** & \textbf{3.70 ± 0.38}** & 5.30 ± 2.12\hphantom{*}\hphantom{*}\\
+ & REVISE & 0.40 ± 0.03\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 118.36 ± 1.68\hphantom{*}\hphantom{*} & \textbf{0.71 ± 0.37}\hphantom{*}\hphantom{*} & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}**\\
- & ECCCo (no CP) & 0.81 ± 0.26\hphantom{*}\hphantom{*} & 79.70 ± 0.57** & 0.74 ± 0.17*\hphantom{*} & 83.73 ± 0.29** & 4.53 ± 0.49\hphantom{*}\hphantom{*} & 5.05 ± 2.96\hphantom{*}\hphantom{*}\\
+ & Schut & 0.36 ± 0.05\hphantom{*}\hphantom{*} & 0.28 ± 0.09\hphantom{*}\hphantom{*} & 114.37 ± 1.21\hphantom{*}\hphantom{*} & 1.32 ± 0.72\hphantom{*}\hphantom{*} & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03*\hphantom{*}\\
- & ECCCo (no EBM) & 0.89 ± 0.26\hphantom{*}\hphantom{*} & 90.34 ± 0.35\hphantom{*}\hphantom{*} & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 71.82 ± 0.18** & 5.00 ± 0.77\hphantom{*}\hphantom{*} & 5.20 ± 2.19\hphantom{*}\hphantom{*}\\
-
- & REVISE & \textbf{0.61 ± 0.09}** & 117.40 ± 1.20\hphantom{*}\hphantom{*} & \textbf{0.68 ± 0.31}*\hphantom{*} & 94.71 ± 1.61** & 3.83 ± 0.60*\hphantom{*} & 5.56 ± 2.24\hphantom{*}\hphantom{*}\\
-
- & Schut & 1.09 ± 0.21\hphantom{*}\hphantom{*} & 113.59 ± 0.49\hphantom{*}\hphantom{*} & 1.03 ± 0.27\hphantom{*}\hphantom{*} & 84.61 ± 0.44** & 4.95 ± 0.71\hphantom{*}\hphantom{*} & 6.43 ± 2.81\hphantom{*}\hphantom{*}\\
-
-\multirow{-10}{*}{\raggedright\arraybackslash JEM} & Wachter & 0.88 ± 0.26\hphantom{*}\hphantom{*} & 83.19 ± 0.37\hphantom{*}\hphantom{*} & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 110.44 ± 0.42\hphantom{*}\hphantom{*} & 5.00 ± 0.76\hphantom{*}\hphantom{*} & 6.23 ± 1.45\hphantom{*}\hphantom{*}\\
+\multirow{-7}{*}{\raggedright\arraybackslash MLP} & Wachter & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & 84.37 ± 0.99\hphantom{*}\hphantom{*} & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.37 ± 0.04\hphantom{*}\hphantom{*}\\
\cmidrule{1-8}
- & ECCCo-L1 & 1.01 ± 0.77\hphantom{*}\hphantom{*} & 43.76 ± 0.16** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 35.19 ± 0.09** & 4.81 ± 0.64\hphantom{*}\hphantom{*} & 2.38 ± 0.34**\\
+ & ECCCo & 0.02 ± 0.01** & 0.05 ± 0.01** & \textbf{74.71 ± 3.49}** & 0.82 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.22 ± 0.02}*\hphantom{*} & 0.46 ± 0.02\hphantom{*}\hphantom{*}\\
- & ECCCo-L1 (no CP) & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 39.79 ± 0.19** & 0.99 ± 0.33\hphantom{*}\hphantom{*} & 42.06 ± 0.12** & 4.83 ± 0.62\hphantom{*}\hphantom{*} & 2.57 ± 0.49**\\
+ & ECCCo+ & & & 97.53 ± 0.20\hphantom{*}\hphantom{*} & 0.72 ± 0.15*\hphantom{*} & 0.22 ± 0.02*\hphantom{*} & 0.36 ± 0.03*\hphantom{*}\\
- & ECCCo-L1 (no EBM) & 1.00 ± 0.77\hphantom{*}\hphantom{*} & \textbf{33.79 ± 0.23}** & 0.98 ± 0.32\hphantom{*}\hphantom{*} & \textbf{30.52 ± 0.16}** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & \textbf{2.38 ± 0.35}**\\
+ & ECCCo (no CP) & \textbf{0.02 ± 0.01}** & \textbf{0.05 ± 0.01}** & 79.70 ± 0.57** & 0.81 ± 0.26\hphantom{*}\hphantom{*} & & \\
- & ECCCo & 2.03 ± 1.30\hphantom{*}\hphantom{*} & 80.15 ± 1.86** & 3.41 ± 2.28\hphantom{*}\hphantom{*} & 67.91 ± 1.63** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 6.58 ± 1.69\hphantom{*}\hphantom{*}\\
+ & ECCCo (no EBM) & 0.09 ± 0.05\hphantom{*}\hphantom{*} & 0.11 ± 0.09\hphantom{*}\hphantom{*} & 90.34 ± 0.35\hphantom{*}\hphantom{*} & 0.89 ± 0.26\hphantom{*}\hphantom{*} & & \\
- & ECCCo+ & 1.64 ± 1.01\hphantom{*}\hphantom{*} & 98.25 ± 0.57\hphantom{*}\hphantom{*} & 2.71 ± 2.32\hphantom{*}\hphantom{*} & 82.72 ± 1.12** & 3.79 ± 0.39** & 6.80 ± 1.65\hphantom{*}\hphantom{*}\\
+ & REVISE & 0.15 ± 0.04\hphantom{*}\hphantom{*} & 0.18 ± 0.01\hphantom{*}\hphantom{*} & 117.40 ± 1.20\hphantom{*}\hphantom{*} & \textbf{0.61 ± 0.09}** & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.03}**\\
- & ECCCo (no CP) & 2.02 ± 1.30\hphantom{*}\hphantom{*} & 82.52 ± 1.18*\hphantom{*} & 3.40 ± 2.28\hphantom{*}\hphantom{*} & 88.72 ± 2.28** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 6.82 ± 1.56\hphantom{*}\hphantom{*}\\
+ & Schut & 0.25 ± 0.16\hphantom{*}\hphantom{*} & 0.37 ± 0.31\hphantom{*}\hphantom{*} & 113.59 ± 0.49\hphantom{*}\hphantom{*} & 1.09 ± 0.21\hphantom{*}\hphantom{*} & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03*\hphantom{*}\\
+
+\multirow{-7}{*}{\raggedright\arraybackslash JEM} & Wachter & 0.09 ± 0.05\hphantom{*}\hphantom{*} & 0.11 ± 0.09\hphantom{*}\hphantom{*} & 83.19 ± 0.37\hphantom{*}\hphantom{*} & 0.88 ± 0.26\hphantom{*}\hphantom{*} & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.04\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo & & & & & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.03\hphantom{*}\hphantom{*}\\
- & ECCCo (no EBM) & 1.00 ± 0.77\hphantom{*}\hphantom{*} & 92.86 ± 1.05\hphantom{*}\hphantom{*} & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 75.47 ± 1.60** & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 6.65 ± 1.87\hphantom{*}\hphantom{*}\\
+ & ECCCo+ & & & & & \textbf{0.24 ± 0.01}\hphantom{*}\hphantom{*} & 0.33 ± 0.02\hphantom{*}\hphantom{*}\\
- & REVISE & \textbf{0.71 ± 0.37}\hphantom{*}\hphantom{*} & 118.36 ± 1.68\hphantom{*}\hphantom{*} & \textbf{0.64 ± 0.19}*\hphantom{*} & 98.98 ± 0.23** & \textbf{3.70 ± 0.23}** & 6.78 ± 0.40\hphantom{*}\hphantom{*}\\
+ & REVISE & & & & & 0.25 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}*\hphantom{*}\\
- & Schut & 1.32 ± 0.72\hphantom{*}\hphantom{*} & 114.37 ± 1.21\hphantom{*}\hphantom{*} & 1.02 ± 0.31\hphantom{*}\hphantom{*} & 87.66 ± 2.05** & 4.92 ± 0.71\hphantom{*}\hphantom{*} & 7.86 ± 1.41\hphantom{*}\hphantom{*}\\
+ & Schut & & & & & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*}\\
-\multirow{-10}{*}{\raggedright\arraybackslash MLP} & Wachter & 0.99 ± 0.77\hphantom{*}\hphantom{*} & 84.37 ± 0.99\hphantom{*}\hphantom{*} & 0.98 ± 0.32\hphantom{*}\hphantom{*} & 114.38 ± 2.14\hphantom{*}\hphantom{*} & 4.84 ± 0.66\hphantom{*}\hphantom{*} & 6.58 ± 2.00\hphantom{*}\hphantom{*}\\
+\multirow{-5}{*}{\raggedright\arraybackslash LeNet-5} & Wachter & & & & & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.03\hphantom{*}\hphantom{*}\\
\bottomrule
-\end{tabu}}
-\end{table}
+\end{tabular}}
+\end{table*}
diff --git a/paper/contents/table-mnist-valid.tex b/paper/contents/table-mnist-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..d7513c681bafc5aa7da589dde44060ce368a09a4
--- /dev/null
+++ b/paper/contents/table-mnist-valid.tex
@@ -0,0 +1,71 @@
+\begin{table}
+
+\caption{All results for MNIST dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-mnist-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 4.50 ± 0.00** & 131.68 ± 9.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.21 ± 0.02*\hphantom{*} & 0.46 ± 0.02\hphantom{*}\hphantom{*} & 4.50 ± 0.00** & 575.77 ± 91.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.22 ± 0.02*\hphantom{*} & 0.36 ± 0.03*\hphantom{*} & \textbf{4.50 ± 0.00}** & 387.38 ± 53.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.03}** & 4.52 ± 0.01\hphantom{*}\hphantom{*} & 169.18 ± 55.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.32 ± 0.04** & 4.50 ± 0.00\hphantom{*}\hphantom{*} & \textbf{5.82 ± 2.72}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.04\hphantom{*}\hphantom{*} & 4.50 ± 0.01\hphantom{*}\hphantom{*} & 71.42 ± 33.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.02\hphantom{*}\hphantom{*} & 1.93 ± 0.31*\hphantom{*} & 138.43 ± 14.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.21 ± 0.02*\hphantom{*} & 0.45 ± 0.02\hphantom{*}\hphantom{*} & \textbf{1.54 ± 0.06}** & 453.08 ± 34.69\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.21 ± 0.02*\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*} & 1.55 ± 0.09** & 349.56 ± 43.87\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.23 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}*\hphantom{*} & 2.37 ± 0.28\hphantom{*}\hphantom{*} & 165.07 ± 51.64\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.31 ± 0.03*\hphantom{*} & 2.43 ± 0.35\hphantom{*}\hphantom{*} & \textbf{7.10 ± 2.68}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.03\hphantom{*}\hphantom{*} & 2.40 ± 0.29\hphantom{*}\hphantom{*} & 33.85 ± 14.52\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 156.85 ± 17.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 202.03 ± 30.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.33 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 212.68 ± 28.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.30 ± 0.03*\hphantom{*} & 0.26 ± 1.46\hphantom{*}\hphantom{*} & 156.02 ± 34.77\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.01*\hphantom{*} & \textbf{0.27 ± 0.04}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{8.60 ± 0.85}** & \textbf{0.98 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash LeNet-5} & Wachter & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 61.17 ± 21.04\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 0.86 ± 0.00** & 149.69 ± 19.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.22 ± 0.01** & 0.42 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.86 ± 0.00}** & 226.18 ± 24.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.23 ± 0.01*\hphantom{*} & 0.32 ± 0.02*\hphantom{*} & 0.86 ± 0.00** & 218.13 ± 30.72\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}*\hphantom{*} & 0.88 ± 0.01\hphantom{*}\hphantom{*} & 152.21 ± 33.88\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.00\hphantom{*}\hphantom{*} & 0.31 ± 0.03*\hphantom{*} & 1.16 ± 0.34\hphantom{*}\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.03\hphantom{*}\hphantom{*} & 0.88 ± 0.00\hphantom{*}\hphantom{*} & 45.99 ± 18.59\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 0.33 ± 0.01** & 159.62 ± 22.02\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.22 ± 0.01** & 0.41 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.33 ± 0.00}** & 190.61 ± 23.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.23 ± 0.01*\hphantom{*} & 0.32 ± 0.03*\hphantom{*} & 0.33 ± 0.00** & 215.26 ± 32.71\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.30 ± 0.03*\hphantom{*} & 0.37 ± 0.03\hphantom{*}\hphantom{*} & 160.54 ± 37.71\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.24 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.28 ± 0.01}** & 0.63 ± 0.42\hphantom{*}\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.03\hphantom{*}\hphantom{*} & 0.39 ± 0.05\hphantom{*}\hphantom{*} & 56.11 ± 20.82\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-mnist.tex b/paper/contents/table-mnist.tex
new file mode 100644
index 0000000000000000000000000000000000000000..8f726f7def714e0e36d984ba788f23dc0d5a5ae0
--- /dev/null
+++ b/paper/contents/table-mnist.tex
@@ -0,0 +1,71 @@
+\begin{table}
+
+\caption{All results for MNIST dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-mnist} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 4.50 ± 0.00** & 131.68 ± 9.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.22 ± 0.02*\hphantom{*} & 0.46 ± 0.02\hphantom{*}\hphantom{*} & 4.50 ± 0.00** & 574.50 ± 87.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.84 ± 0.37\hphantom{*}\hphantom{*}\\
+
+ & ECCCo+ & 0.22 ± 0.02*\hphantom{*} & 0.36 ± 0.03*\hphantom{*} & 4.50 ± 0.00** & 386.23 ± 54.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.14\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.31 ± 0.03}** & 4.52 ± 0.01\hphantom{*}\hphantom{*} & 168.06 ± 53.58\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.93 ± 0.26\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03*\hphantom{*} & \textbf{1.41 ± 2.11}*\hphantom{*} & \textbf{9.08 ± 2.14}** & \textbf{0.99 ± 0.00}** & 0.22 ± 0.42\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.40 ± 0.04\hphantom{*}\hphantom{*} & 4.51 ± 0.01\hphantom{*}\hphantom{*} & 70.81 ± 32.04\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.91 ± 0.29\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.01}** & 0.43 ± 0.02\hphantom{*}\hphantom{*} & 1.93 ± 0.31*\hphantom{*} & 138.43 ± 14.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.21 ± 0.02*\hphantom{*} & 0.45 ± 0.02\hphantom{*}\hphantom{*} & 1.57 ± 0.15** & 449.75 ± 35.92\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.93 ± 0.26*\hphantom{*}\\
+
+ & ECCCo+ & 0.21 ± 0.02*\hphantom{*} & 0.34 ± 0.03*\hphantom{*} & 1.55 ± 0.09** & 349.56 ± 43.87\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & REVISE & 0.23 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}** & 2.37 ± 0.28\hphantom{*}\hphantom{*} & 165.54 ± 51.39\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.98 ± 0.14**\\
+
+ & Schut & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.22 ± 0.70}** & \textbf{9.77 ± 1.06}** & \textbf{0.99 ± 0.00}** & 0.08 ± 0.27\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.36 ± 0.04\hphantom{*}\hphantom{*} & 2.44 ± 0.27\hphantom{*}\hphantom{*} & 46.85 ± 22.56\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 0.56 ± 0.50\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 156.85 ± 17.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.39 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 202.03 ± 30.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo+ & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.33 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 212.68 ± 28.09\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & REVISE & 0.25 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}*\hphantom{*} & 0.26 ± 1.46\hphantom{*}\hphantom{*} & 156.02 ± 34.77\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & \textbf{9.97 ± 0.21}** & \textbf{0.98 ± 0.00}** & 0.02 ± 0.14\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash LeNet-5} & Wachter & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.35 ± 0.03\hphantom{*}\hphantom{*} & 0.09 ± 0.85\hphantom{*}\hphantom{*} & 65.58 ± 22.50\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 0.62 ± 0.49\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 0.86 ± 0.00** & 149.69 ± 19.19\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.22 ± 0.01** & 0.42 ± 0.02\hphantom{*}\hphantom{*} & 0.86 ± 0.00** & 226.18 ± 24.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo+ & 0.23 ± 0.01*\hphantom{*} & 0.32 ± 0.02*\hphantom{*} & 0.86 ± 0.00** & 218.13 ± 30.72\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}** & 0.88 ± 0.01\hphantom{*}\hphantom{*} & 151.99 ± 33.79\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.99 ± 0.10**\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03*\hphantom{*} & \textbf{0.14 ± 0.39}*\hphantom{*} & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 0.04 ± 0.20\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.37 ± 0.04\hphantom{*}\hphantom{*} & 0.88 ± 0.00\hphantom{*}\hphantom{*} & 55.50 ± 26.76\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 0.76 ± 0.43\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.09 ± 0.00}** & 0.44 ± 0.02\hphantom{*}\hphantom{*} & 0.33 ± 0.01** & 159.62 ± 22.02\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo & 0.22 ± 0.01** & 0.41 ± 0.02\hphantom{*}\hphantom{*} & 0.33 ± 0.00** & 190.61 ± 23.95\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & ECCCo+ & 0.23 ± 0.01*\hphantom{*} & 0.32 ± 0.03*\hphantom{*} & 0.33 ± 0.02** & 215.98 ± 33.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.99 ± 0.10**\\
+
+ & REVISE & 0.24 ± 0.01\hphantom{*}\hphantom{*} & \textbf{0.30 ± 0.03}** & 0.37 ± 0.03\hphantom{*}\hphantom{*} & 160.54 ± 37.71\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & Schut & 0.25 ± 0.01\hphantom{*}\hphantom{*} & 0.34 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.03 ± 0.16}** & \textbf{10.00 ± 0.00}** & \textbf{0.99 ± 0.00}** & 0.02 ± 0.14\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-6}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.24 ± 0.01\hphantom{*}\hphantom{*} & 0.36 ± 0.04\hphantom{*}\hphantom{*} & 0.41 ± 0.06\hphantom{*}\hphantom{*} & 64.78 ± 26.67\hphantom{*}\hphantom{*} & 0.01 ± 0.00\hphantom{*}\hphantom{*} & 0.76 ± 0.43\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-moons-valid.tex b/paper/contents/table-moons-valid.tex
new file mode 100644
index 0000000000000000000000000000000000000000..9cc6e021946a1b1c45a18ffd762336e2bbbbc4bd
--- /dev/null
+++ b/paper/contents/table-moons-valid.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Moons dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-moons-valid} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.23 ± 0.05** & 0.25 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.21 ± 0.15** & 0.25 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.20 ± 0.06}** & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.33 ± 0.28*\hphantom{*} & \textbf{0.15 ± 0.11}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.62 ± 0.21\hphantom{*}\hphantom{*} & 0.15 ± 0.11*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.62 ± 0.22\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.81 ± 0.24\hphantom{*}\hphantom{*} & 0.27 ± 0.12\hphantom{*}\hphantom{*} & 0.28 ± 0.27\hphantom{*}\hphantom{*} & 0.93 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.66 ± 0.33\hphantom{*}\hphantom{*} & 0.24 ± 0.13\hphantom{*}\hphantom{*} & 0.06 ± 0.20\hphantom{*}\hphantom{*} & 0.88 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.10 ± 0.20}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.68 ± 0.25\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{0.85 ± 0.27}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.21 ± 0.10}** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.01 ± 0.11*\hphantom{*} & 1.03 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.23 ± 0.13** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.10 ± 0.22\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.24 ± 0.13** & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.54 ± 0.22\hphantom{*}\hphantom{*} & 0.12 ± 0.01** & \textbf{0.00 ± 0.00}** & 1.66 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.37 ± 0.22*\hphantom{*} & \textbf{0.12 ± 0.01}** & \textbf{0.00 ± 0.00}** & 1.65 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.66 ± 0.23\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.76 ± 0.19\hphantom{*}\hphantom{*} & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.67 ± 0.12\hphantom{*}\hphantom{*} & \textbf{0.98 ± 0.22}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.65 ± 0.25\hphantom{*}\hphantom{*} & 0.17 ± 0.17\hphantom{*}\hphantom{*} & 0.20 ± 0.32\hphantom{*}\hphantom{*} & 0.99 ± 0.06\hphantom{*}\hphantom{*} & \textbf{0.09 ± 0.19}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.59 ± 0.17\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.14 ± 0.25\hphantom{*}\hphantom{*} & 1.01 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.41 ± 0.13** & 0.27 ± 0.11\hphantom{*}\hphantom{*} & 0.25 ± 0.25\hphantom{*}\hphantom{*} & 0.92 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.36 ± 0.15** & \textbf{0.26 ± 0.12}\hphantom{*}\hphantom{*} & 0.19 ± 0.24\hphantom{*}\hphantom{*} & 0.93 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.34 ± 0.13}** & 0.28 ± 0.12\hphantom{*}\hphantom{*} & 0.30 ± 0.25\hphantom{*}\hphantom{*} & 0.90 ± 0.28\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 1.95 ± 1.52\hphantom{*}\hphantom{*} & 3.06 ± 2.55\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 5.29 ± 3.62\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 1.92 ± 1.78\hphantom{*}\hphantom{*} & 3.06 ± 2.55\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 5.29 ± 3.62\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 1.25 ± 0.29\hphantom{*}\hphantom{*} & 0.28 ± 0.12\hphantom{*}\hphantom{*} & 0.30 ± 0.25\hphantom{*}\hphantom{*} & 0.90 ± 0.28\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.93 ± 0.49\hphantom{*}\hphantom{*} & 0.27 ± 0.08\hphantom{*}\hphantom{*} & 0.28 ± 0.25\hphantom{*}\hphantom{*} & 0.96 ± 0.28\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 1.06 ± 0.36\hphantom{*}\hphantom{*} & 0.30 ± 0.07\hphantom{*}\hphantom{*} & 0.04 ± 0.18*\hphantom{*} & \textbf{0.71 ± 0.18}*\hphantom{*} & \textbf{0.27 ± 0.25}*\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 1.23 ± 0.32\hphantom{*}\hphantom{*} & 0.28 ± 0.12\hphantom{*}\hphantom{*} & 0.30 ± 0.25\hphantom{*}\hphantom{*} & 0.90 ± 0.29\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.42 ± 0.10** & 0.28 ± 0.15\hphantom{*}\hphantom{*} & 0.21 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.31 ± 0.17** & 0.28 ± 0.15\hphantom{*}\hphantom{*} & 0.21 ± 0.25\hphantom{*}\hphantom{*} & 0.83 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.31 ± 0.16}** & 0.28 ± 0.15\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 0.82 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 1.82 ± 1.28\hphantom{*}\hphantom{*} & 3.09 ± 2.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 5.17 ± 2.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 1.78 ± 1.34\hphantom{*}\hphantom{*} & 3.09 ± 2.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 5.17 ± 2.94\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.97 ± 0.42\hphantom{*}\hphantom{*} & 0.28 ± 0.15\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 0.82 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.88 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.07}\hphantom{*}\hphantom{*} & 0.31 ± 0.26\hphantom{*}\hphantom{*} & 1.00 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & Schut & 1.07 ± 0.37\hphantom{*}\hphantom{*} & 0.28 ± 0.10\hphantom{*}\hphantom{*} & 0.06 ± 0.17*\hphantom{*} & \textbf{0.72 ± 0.18}\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.25}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 1.17 ± 0.28\hphantom{*}\hphantom{*} & 0.29 ± 0.15\hphantom{*}\hphantom{*} & 0.23 ± 0.26\hphantom{*}\hphantom{*} & 0.82 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-moons.tex b/paper/contents/table-moons.tex
new file mode 100644
index 0000000000000000000000000000000000000000..b9f963d61cc30fc648756dd102011b9684812586
--- /dev/null
+++ b/paper/contents/table-moons.tex
@@ -0,0 +1,83 @@
+\begin{table}
+
+\caption{All results for Moons dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-moons} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{cccccccc}
+\toprule
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+ & ECCCo-L1 & 0.23 ± 0.05** & 0.25 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.21 ± 0.15** & 0.25 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.20 ± 0.06}** & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.33 ± 0.28*\hphantom{*} & \textbf{0.15 ± 0.11}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.62 ± 0.21\hphantom{*}\hphantom{*} & 0.15 ± 0.11*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.62 ± 0.22\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.81 ± 0.24\hphantom{*}\hphantom{*} & 0.27 ± 0.12\hphantom{*}\hphantom{*} & 0.28 ± 0.27\hphantom{*}\hphantom{*} & 0.93 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 0.26 ± 0.17\hphantom{*}\hphantom{*} & 0.06 ± 0.20\hphantom{*}\hphantom{*} & 0.86 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.21}\hphantom{*}\hphantom{*} & 0.97 ± 0.17\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.68 ± 0.25\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{0.85 ± 0.27}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.21 ± 0.10}** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.01 ± 0.11*\hphantom{*} & 1.03 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.23 ± 0.13** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.10 ± 0.22\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.24 ± 0.13** & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 0.54 ± 0.22\hphantom{*}\hphantom{*} & 0.12 ± 0.01** & \textbf{0.00 ± 0.00}** & 1.66 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 0.37 ± 0.22*\hphantom{*} & \textbf{0.12 ± 0.01}** & \textbf{0.00 ± 0.00}** & 1.65 ± 0.31\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 0.66 ± 0.23\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.76 ± 0.19\hphantom{*}\hphantom{*} & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.67 ± 0.12\hphantom{*}\hphantom{*} & \textbf{0.98 ± 0.22}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+
+ & Schut & 0.65 ± 0.25\hphantom{*}\hphantom{*} & 0.17 ± 0.17\hphantom{*}\hphantom{*} & 0.20 ± 0.31\hphantom{*}\hphantom{*} & 0.99 ± 0.06\hphantom{*}\hphantom{*} & \textbf{0.09 ± 0.19}\hphantom{*}\hphantom{*} & 0.99 ± 0.10\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.59 ± 0.17\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.14 ± 0.25\hphantom{*}\hphantom{*} & 1.01 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & \textbf{0.40 ± 0.17}** & 0.79 ± 0.91\hphantom{*}\hphantom{*} & 0.19 ± 0.24\hphantom{*}\hphantom{*} & 1.23 ± 0.61\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.47 ± 0.26** & 0.78 ± 0.91\hphantom{*}\hphantom{*} & 0.14 ± 0.23\hphantom{*}\hphantom{*} & 1.24 ± 0.60\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & 0.44 ± 0.25** & 0.80 ± 0.92\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.23 ± 0.63\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 2.03 ± 1.38\hphantom{*}\hphantom{*} & 2.99 ± 2.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.66 ± 3.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 2.02 ± 1.57\hphantom{*}\hphantom{*} & 2.98 ± 2.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.66 ± 3.32\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 1.57 ± 0.60\hphantom{*}\hphantom{*} & 0.80 ± 0.92\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.23 ± 0.63\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.93 ± 0.49*\hphantom{*} & \textbf{0.27 ± 0.08}** & 0.28 ± 0.25\hphantom{*}\hphantom{*} & 0.96 ± 0.28\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & Schut & 1.04 ± 0.47\hphantom{*}\hphantom{*} & 0.54 ± 0.40\hphantom{*}\hphantom{*} & 0.03 ± 0.15*\hphantom{*} & \textbf{0.68 ± 0.20}** & \textbf{0.20 ± 0.25}\hphantom{*}\hphantom{*} & 0.70 ± 0.46\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 1.50 ± 0.59\hphantom{*}\hphantom{*} & 0.81 ± 0.91\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.22 ± 0.63\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+\cmidrule{1-8}
+ & ECCCo-L1 & 0.41 ± 0.13** & 0.43 ± 0.43\hphantom{*}\hphantom{*} & 0.18 ± 0.24\hphantom{*}\hphantom{*} & 0.85 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no CP) & 0.35 ± 0.20** & 0.43 ± 0.43\hphantom{*}\hphantom{*} & 0.18 ± 0.25\hphantom{*}\hphantom{*} & 0.85 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo-L1 (no EBM) & \textbf{0.33 ± 0.18}** & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.20 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & ECCCo & 1.79 ± 1.23\hphantom{*}\hphantom{*} & 2.92 ± 2.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.70 ± 3.08\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.89 ± 0.31\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no CP) & 1.72 ± 1.30\hphantom{*}\hphantom{*} & 2.92 ± 2.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.70 ± 3.08\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.89 ± 0.31\hphantom{*}\hphantom{*}\\
+
+ & ECCCo (no EBM) & 1.01 ± 0.45\hphantom{*}\hphantom{*} & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.20 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.21\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+
+ & REVISE & 0.88 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.07}** & 0.31 ± 0.26\hphantom{*}\hphantom{*} & 1.00 ± 0.27\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.00 ± 0.00}**\\
+
+ & Schut & 1.06 ± 0.41\hphantom{*}\hphantom{*} & 0.34 ± 0.21\hphantom{*}\hphantom{*} & 0.06 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.67 ± 0.23}\hphantom{*}\hphantom{*} & \textbf{0.23 ± 0.25}\hphantom{*}\hphantom{*} & 0.90 ± 0.30\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 1.27 ± 0.38\hphantom{*}\hphantom{*} & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.21 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.88 ± 0.33\hphantom{*}\hphantom{*}\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table-synthetic.tex b/paper/contents/table-synthetic.tex
index df1746fbf5c7717ecbfc890e2f0e7551ab9834d8..c2e4fb3506eb7a5d8b6edaf7d6bae8018739a9cd 100644
--- a/paper/contents/table-synthetic.tex
+++ b/paper/contents/table-synthetic.tex
@@ -1,37 +1,232 @@
-\begin{table}
-\caption{Results for synthetic datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-synthetic} \newline}
-\centering
-\resizebox{\linewidth}{!}{
-\begin{tabular}[t]{llcccccc}
+\begin{longtable}[t]{ccccccccc}
+\caption{All results for all datasets: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-full} \newline}\\
\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{Linearly Separable} & \multicolumn{2}{c}{Moons} & \multicolumn{2}{c}{Circles} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓\\
+Data & Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
\midrule
- & ECCCo & \textbf{0.03 ± 0.06}** & \textbf{0.20 ± 0.08}** & \textbf{0.31 ± 0.30}*\hphantom{*} & \textbf{1.20 ± 0.15}** & 0.52 ± 0.36\hphantom{*}\hphantom{*} & 1.22 ± 0.46\hphantom{*}\hphantom{*}\\
+\endfirsthead
+\caption[]{All results for all datasets: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \newline \textit{(continued)}}\\
+\toprule
+Data & Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Cost ↓ & Redundancy ↑ & Validity ↑\\
+\midrule
+\endhead
+
+\endfoot
+\bottomrule
+\endlastfoot
+ & & ECCCo-L1 & 0.19 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & ECCCo (no CP) & 0.03 ± 0.06** & 0.20 ± 0.08** & 0.37 ± 0.30*\hphantom{*} & 1.21 ± 0.17** & 0.54 ± 0.39\hphantom{*}\hphantom{*} & 1.21 ± 0.46\hphantom{*}\hphantom{*}\\
+ & & ECCCo-L1 (no CP) & 0.18 ± 0.22*\hphantom{*} & 0.51 ± 0.22\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & ECCCo (no EBM) & 0.16 ± 0.11\hphantom{*}\hphantom{*} & 0.34 ± 0.19\hphantom{*}\hphantom{*} & 0.91 ± 0.32\hphantom{*}\hphantom{*} & 1.71 ± 0.25\hphantom{*}\hphantom{*} & 0.70 ± 0.33\hphantom{*}\hphantom{*} & 1.30 ± 0.37\hphantom{*}\hphantom{*}\\
+ & & ECCCo-L1 (no EBM) & \textbf{0.17 ± 0.20}*\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & REVISE & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.41 ± 0.01** & 0.78 ± 0.23\hphantom{*}\hphantom{*} & 1.57 ± 0.26\hphantom{*}\hphantom{*} & \textbf{0.48 ± 0.16}*\hphantom{*} & \textbf{0.95 ± 0.32}*\hphantom{*}\\
+ & & ECCCo & 0.42 ± 0.51\hphantom{*}\hphantom{*} & 0.48 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.10 ± 0.18\hphantom{*}\hphantom{*} & -0.01 ± 0.05\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & Schut & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.73 ± 0.17\hphantom{*}\hphantom{*} & 0.67 ± 0.27\hphantom{*}\hphantom{*} & 1.50 ± 0.22*\hphantom{*} & 0.54 ± 0.43\hphantom{*}\hphantom{*} & 1.28 ± 0.53\hphantom{*}\hphantom{*}\\
+ & & ECCCo (no CP) & 0.50 ± 0.57\hphantom{*}\hphantom{*} & \textbf{0.48 ± 0.20}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.10 ± 0.18\hphantom{*}\hphantom{*} & -0.01 ± 0.05\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\multirow{-6}{*}{\raggedright\arraybackslash JEM} & Wachter & 0.18 ± 0.10\hphantom{*}\hphantom{*} & 0.44 ± 0.17\hphantom{*}\hphantom{*} & 0.80 ± 0.27\hphantom{*}\hphantom{*} & 1.78 ± 0.24\hphantom{*}\hphantom{*} & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 1.33 ± 0.32\hphantom{*}\hphantom{*}\\
-\cmidrule{1-8}
- & ECCCo & \textbf{0.29 ± 0.05}** & 0.23 ± 0.06** & 0.80 ± 0.62\hphantom{*}\hphantom{*} & 1.69 ± 0.40\hphantom{*}\hphantom{*} & 0.65 ± 0.53\hphantom{*}\hphantom{*} & 1.17 ± 0.41\hphantom{*}\hphantom{*}\\
+ & & ECCCo (no EBM) & 0.36 ± 0.46\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & ECCCo (no CP) & 0.29 ± 0.05** & \textbf{0.23 ± 0.07}** & \textbf{0.79 ± 0.62}\hphantom{*}\hphantom{*} & 1.68 ± 0.42\hphantom{*}\hphantom{*} & \textbf{0.49 ± 0.35}\hphantom{*}\hphantom{*} & 1.19 ± 0.44\hphantom{*}\hphantom{*}\\
+ & & REVISE & 0.30 ± 0.36\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.05 ± 0.34}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.63 ± 0.49}\hphantom{*}\hphantom{*}\\
- & ECCCo (no EBM) & 0.46 ± 0.05\hphantom{*}\hphantom{*} & 0.28 ± 0.04** & 1.34 ± 0.47\hphantom{*}\hphantom{*} & 1.68 ± 0.47\hphantom{*}\hphantom{*} & 0.84 ± 0.51\hphantom{*}\hphantom{*} & 1.23 ± 0.31\hphantom{*}\hphantom{*}\\
+ & & Schut & 0.49 ± 0.67\hphantom{*}\hphantom{*} & 0.75 ± 0.43\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.70 ± 0.55\hphantom{*}\hphantom{*} & -0.01 ± 0.05\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & REVISE & 0.56 ± 0.05\hphantom{*}\hphantom{*} & 0.41 ± 0.01\hphantom{*}\hphantom{*} & 1.45 ± 0.44\hphantom{*}\hphantom{*} & \textbf{1.64 ± 0.31}\hphantom{*}\hphantom{*} & 0.58 ± 0.52\hphantom{*}\hphantom{*} & \textbf{0.95 ± 0.32}\hphantom{*}\hphantom{*}\\
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.41 ± 0.50\hphantom{*}\hphantom{*} & 0.51 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.14 ± 0.20\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.21 ± 0.24*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
- & Schut & 0.43 ± 0.06*\hphantom{*} & 0.47 ± 0.36\hphantom{*}\hphantom{*} & 1.45 ± 0.55\hphantom{*}\hphantom{*} & 1.73 ± 0.48\hphantom{*}\hphantom{*} & 0.58 ± 0.37\hphantom{*}\hphantom{*} & 1.23 ± 0.43\hphantom{*}\hphantom{*}\\
+ & & ECCCo-L1 (no CP) & 0.21 ± 0.23*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.42\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
-\multirow{-6}{*}{\raggedright\arraybackslash MLP} & Wachter & 0.51 ± 0.04\hphantom{*}\hphantom{*} & 0.40 ± 0.08\hphantom{*}\hphantom{*} & 1.32 ± 0.41\hphantom{*}\hphantom{*} & 1.69 ± 0.32\hphantom{*}\hphantom{*} & 0.83 ± 0.50\hphantom{*}\hphantom{*} & 1.24 ± 0.29\hphantom{*}\hphantom{*}\\
-\bottomrule
-\end{tabular}}
-\end{table}
+ & & ECCCo-L1 (no EBM) & \textbf{0.20 ± 0.23}*\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.52 ± 0.61\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.50 ± 0.58\hphantom{*}\hphantom{*} & 0.61 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.26 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.51 ± 0.57\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.31 ± 0.34\hphantom{*}\hphantom{*} & \textbf{0.60 ± 0.23}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{1.04 ± 0.34}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.63 ± 0.49}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.67 ± 0.91\hphantom{*}\hphantom{*} & 1.14 ± 0.96\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 2.30 ± 1.32\hphantom{*}\hphantom{*} & -0.01 ± 0.07\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.52 ± 0.59\hphantom{*}\hphantom{*} & 0.60 ± 0.42\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.25 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.43 ± 0.11\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & \textbf{0.15 ± 0.18}*\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.39 ± 0.46\hphantom{*}\hphantom{*} & 0.36 ± 0.06*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.91 ± 0.17}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.40 ± 0.46\hphantom{*}\hphantom{*} & \textbf{0.36 ± 0.06}*\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.91 ± 0.17\hphantom{*}\hphantom{*} & -0.01 ± 0.05\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.37 ± 0.45\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.29 ± 0.36\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.04 ± 0.33\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.63 ± 0.49}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.38 ± 0.57\hphantom{*}\hphantom{*} & 0.58 ± 0.20\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.41 ± 0.27\hphantom{*}\hphantom{*} & -0.01 ± 0.05\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.40 ± 0.49\hphantom{*}\hphantom{*} & 0.43 ± 0.12\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.01 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.17 ± 0.20*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.17 ± 0.21*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & \textbf{0.16 ± 0.19}*\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.39 ± 0.47\hphantom{*}\hphantom{*} & \textbf{0.39 ± 0.07}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.97 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.42 ± 0.50\hphantom{*}\hphantom{*} & 0.39 ± 0.07\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.97 ± 0.15}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.39 ± 0.47\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.31 ± 0.35\hphantom{*}\hphantom{*} & 0.60 ± 0.23\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.03 ± 0.32\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.63 ± 0.49}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.39 ± 0.59\hphantom{*}\hphantom{*} & 0.62 ± 0.24\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.48 ± 0.32\hphantom{*}\hphantom{*} & -0.01 ± 0.07\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-36}{*}{\centering\arraybackslash Circles} & \multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.43 ± 0.52\hphantom{*}\hphantom{*} & 0.45 ± 0.13\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 1.07 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-9}
+ & & ECCCo-L1 & 0.03 ± 0.01** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & \textbf{0.03 ± 0.01}** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.04 ± 0.02** & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.07 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.05\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.07 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.08 ± 0.05}\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.92 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.08 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.15 ± 0.04\hphantom{*}\hphantom{*} & 0.17 ± 0.00\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.15}** & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.52 ± 0.50}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.25 ± 0.16\hphantom{*}\hphantom{*} & 0.37 ± 0.30\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.99 ± 0.05\hphantom{*}\hphantom{*} & -0.24 ± 0.25\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.09 ± 0.04\hphantom{*}\hphantom{*} & 0.10 ± 0.06\hphantom{*}\hphantom{*} & 0.00 ± 0.00\hphantom{*}\hphantom{*} & 0.95 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & \textbf{0.03 ± 0.01}** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.03 ± 0.01** & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.08 ± 0.02\hphantom{*}\hphantom{*} & 0.11 ± 0.04\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.02 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.09 ± 0.03\hphantom{*}\hphantom{*} & 0.11 ± 0.05\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.27 ± 0.01\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.36 ± 0.37\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.15}** & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.48 ± 0.50}*\hphantom{*}\\
+
+ & & Schut & 0.33 ± 0.06\hphantom{*}\hphantom{*} & 0.66 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.00\hphantom{*}\hphantom{*} & -0.50 ± 0.00\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.11 ± 0.05}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.00 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & \textbf{0.13 ± 0.01}** & \textbf{0.09 ± 0.03}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.13 ± 0.01** & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.14 ± 0.02** & 0.43 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.58 ± 0.46\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.40 ± 0.03\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.54 ± 0.49\hphantom{*}\hphantom{*} & \textbf{0.42 ± 0.13}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.55 ± 0.50}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.36 ± 0.04\hphantom{*}\hphantom{*} & 0.28 ± 0.10\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.71 ± 0.13\hphantom{*}\hphantom{*} & -0.35 ± 0.23\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 0.32 ± 0.04\hphantom{*}\hphantom{*} & 0.09 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.63 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.12 ± 0.01** & \textbf{0.08 ± 0.02}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.12 ± 0.01** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.12 ± 0.02** & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & \textbf{0.08 ± 0.04}** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.08 ± 0.04** & 0.29 ± 0.03\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.37 ± 0.14\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.40 ± 0.04\hphantom{*}\hphantom{*} & 0.17 ± 0.01\hphantom{*}\hphantom{*} & 0.52 ± 0.49\hphantom{*}\hphantom{*} & \textbf{0.40 ± 0.13}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.53 ± 0.50}\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.37 ± 0.06\hphantom{*}\hphantom{*} & 0.30 ± 0.11\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.70 ± 0.12\hphantom{*}\hphantom{*} & -0.50 ± 0.00\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-36}{*}{\centering\arraybackslash Linearly Separable} & \multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 0.31 ± 0.03\hphantom{*}\hphantom{*} & 0.08 ± 0.02\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.64 ± 0.13\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{1-9}
+ & & ECCCo-L1 & 0.23 ± 0.05** & 0.25 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.21 ± 0.15** & 0.25 ± 0.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.87 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & \textbf{0.20 ± 0.06}** & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.33 ± 0.28*\hphantom{*} & \textbf{0.15 ± 0.11}*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.62 ± 0.21\hphantom{*}\hphantom{*} & 0.15 ± 0.11*\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 1.51 ± 0.61\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.62 ± 0.22\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & 0.85 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.81 ± 0.24\hphantom{*}\hphantom{*} & 0.27 ± 0.12\hphantom{*}\hphantom{*} & 0.28 ± 0.27\hphantom{*}\hphantom{*} & 0.93 ± 0.24\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.68 ± 0.34\hphantom{*}\hphantom{*} & 0.26 ± 0.17\hphantom{*}\hphantom{*} & 0.06 ± 0.20\hphantom{*}\hphantom{*} & 0.86 ± 0.20\hphantom{*}\hphantom{*} & -0.11 ± 0.21\hphantom{*}\hphantom{*} & \textbf{-0.97 ± 0.17}\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM} & Wachter & 0.68 ± 0.25\hphantom{*}\hphantom{*} & 0.25 ± 0.17\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{0.85 ± 0.27}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & \textbf{0.21 ± 0.10}** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.01 ± 0.11*\hphantom{*} & 1.03 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.23 ± 0.13** & 0.16 ± 0.14\hphantom{*}\hphantom{*} & 0.10 ± 0.22\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.24 ± 0.13** & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 0.54 ± 0.22\hphantom{*}\hphantom{*} & 0.12 ± 0.01** & \textbf{0.00 ± 0.00}** & 1.66 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 0.37 ± 0.22*\hphantom{*} & \textbf{0.12 ± 0.01}** & \textbf{0.00 ± 0.00}** & 1.65 ± 0.31\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 0.66 ± 0.23\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.06 ± 0.18\hphantom{*}\hphantom{*} & 1.03 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.76 ± 0.19\hphantom{*}\hphantom{*} & 0.19 ± 0.03\hphantom{*}\hphantom{*} & 0.67 ± 0.12\hphantom{*}\hphantom{*} & \textbf{0.98 ± 0.22}\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 0.65 ± 0.25\hphantom{*}\hphantom{*} & 0.17 ± 0.17\hphantom{*}\hphantom{*} & 0.20 ± 0.31\hphantom{*}\hphantom{*} & 0.99 ± 0.06\hphantom{*}\hphantom{*} & -0.09 ± 0.19\hphantom{*}\hphantom{*} & \textbf{-0.99 ± 0.10}\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash JEM Ensemble} & Wachter & 0.59 ± 0.17\hphantom{*}\hphantom{*} & 0.17 ± 0.14\hphantom{*}\hphantom{*} & 0.14 ± 0.25\hphantom{*}\hphantom{*} & 1.01 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & \textbf{0.40 ± 0.17}** & 0.79 ± 0.91\hphantom{*}\hphantom{*} & 0.19 ± 0.24\hphantom{*}\hphantom{*} & 1.23 ± 0.61\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.47 ± 0.26** & 0.78 ± 0.91\hphantom{*}\hphantom{*} & 0.14 ± 0.23\hphantom{*}\hphantom{*} & 1.24 ± 0.60\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & 0.44 ± 0.25** & 0.80 ± 0.92\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.23 ± 0.63\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 2.03 ± 1.38\hphantom{*}\hphantom{*} & 2.99 ± 2.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.66 ± 3.32\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 2.02 ± 1.57\hphantom{*}\hphantom{*} & 2.98 ± 2.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.66 ± 3.32\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 1.57 ± 0.60\hphantom{*}\hphantom{*} & 0.80 ± 0.92\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.23 ± 0.63\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.93 ± 0.49*\hphantom{*} & \textbf{0.27 ± 0.08}** & 0.28 ± 0.25\hphantom{*}\hphantom{*} & 0.96 ± 0.28\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 1.04 ± 0.47\hphantom{*}\hphantom{*} & 0.54 ± 0.40\hphantom{*}\hphantom{*} & 0.03 ± 0.15*\hphantom{*} & \textbf{0.68 ± 0.20}** & -0.20 ± 0.25\hphantom{*}\hphantom{*} & \textbf{-0.70 ± 0.46}\hphantom{*}\hphantom{*}\\
+
+ & \multirow[t]{-9}{*}{\centering\arraybackslash MLP} & Wachter & 1.50 ± 0.59\hphantom{*}\hphantom{*} & 0.81 ± 0.91\hphantom{*}\hphantom{*} & 0.23 ± 0.25\hphantom{*}\hphantom{*} & 1.22 ± 0.63\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.75 ± 0.44\hphantom{*}\hphantom{*}\\
+\cmidrule{2-9}
+ & & ECCCo-L1 & 0.41 ± 0.13** & 0.43 ± 0.43\hphantom{*}\hphantom{*} & 0.18 ± 0.24\hphantom{*}\hphantom{*} & 0.85 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.88 ± 0.33}\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no CP) & 0.35 ± 0.20** & 0.43 ± 0.43\hphantom{*}\hphantom{*} & 0.18 ± 0.25\hphantom{*}\hphantom{*} & 0.85 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.88 ± 0.33}\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo-L1 (no EBM) & \textbf{0.33 ± 0.18}** & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.20 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.88 ± 0.33}\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo & 1.79 ± 1.23\hphantom{*}\hphantom{*} & 2.92 ± 2.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.70 ± 3.08\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.89 ± 0.31\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no CP) & 1.72 ± 1.30\hphantom{*}\hphantom{*} & 2.92 ± 2.15\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}** & 4.70 ± 3.08\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -0.89 ± 0.31\hphantom{*}\hphantom{*}\\
+
+ & & ECCCo (no EBM) & 1.01 ± 0.45\hphantom{*}\hphantom{*} & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.20 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.21\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.88 ± 0.33}\hphantom{*}\hphantom{*}\\
+
+ & & REVISE & 0.88 ± 0.43\hphantom{*}\hphantom{*} & \textbf{0.25 ± 0.07}** & 0.31 ± 0.26\hphantom{*}\hphantom{*} & 1.00 ± 0.27\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & -1.00 ± 0.00\hphantom{*}\hphantom{*}\\
+
+ & & Schut & 1.06 ± 0.41\hphantom{*}\hphantom{*} & 0.34 ± 0.21\hphantom{*}\hphantom{*} & 0.06 ± 0.16\hphantom{*}\hphantom{*} & \textbf{0.67 ± 0.23}\hphantom{*}\hphantom{*} & -0.23 ± 0.25\hphantom{*}\hphantom{*} & -0.90 ± 0.30\hphantom{*}\hphantom{*}\\
+
+\multirow[t]{-36}{*}{\centering\arraybackslash Moons} & \multirow[t]{-9}{*}{\centering\arraybackslash MLP Ensemble} & Wachter & 1.27 ± 0.38\hphantom{*}\hphantom{*} & 0.43 ± 0.42\hphantom{*}\hphantom{*} & 0.21 ± 0.25\hphantom{*}\hphantom{*} & 0.84 ± 0.22\hphantom{*}\hphantom{*} & \textbf{0.00 ± 0.00}\hphantom{*}\hphantom{*} & \textbf{-0.88 ± 0.33}\hphantom{*}\hphantom{*}\\*
+\end{longtable}
diff --git a/paper/contents/table-tabular.tex b/paper/contents/table-tabular.tex
index 07e730a06836d2598b03004db6d3d882830d26f2..ad962ffed0a62753335530ee708e803e536b6098 100644
--- a/paper/contents/table-tabular.tex
+++ b/paper/contents/table-tabular.tex
@@ -1,31 +1,41 @@
-\begin{table}
+\begin{table*}
-\caption{Results for tabular datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \label{tab:results-tabular} \newline}
+\caption{Results for tabular datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\textit{Wachter}). \label{tab:results-tabular} \newline}
\centering
\resizebox{\linewidth}{!}{
-\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X>{\centering}X}
+\begin{tabular}[t]{llcccccc}
\toprule
-\multicolumn{2}{c}{ } & \multicolumn{2}{c}{GMSC} & \multicolumn{2}{c}{California Housing} & \multicolumn{2}{c}{German Credit} \\
-\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6} \cmidrule(l{3pt}r{3pt}){7-8}
-Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓ & Unfaithfulness ↓ & Implausibility ↓\\
+\multicolumn{2}{c}{ } & \multicolumn{3}{c}{California Housing} & \multicolumn{3}{c}{GMSC} \\
+\cmidrule(l{3pt}r{3pt}){3-5} \cmidrule(l{3pt}r{3pt}){6-8}
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓ & Unfaithfulness ↓ & Implausibility ↓ & Uncertainty ↓\\
\midrule
- & ECCCo & 37.08 ± 0.45\hphantom{*}\hphantom{*} & 1.71 ± 4.31\hphantom{*}\hphantom{*} & 39.38 ± 0.21\hphantom{*}\hphantom{*} & 1.72 ± 2.09\hphantom{*}\hphantom{*} & 2.40 ± 0.33\hphantom{*}\hphantom{*} & 4.75 ± 0.79\hphantom{*}\hphantom{*}\\
+ & ECCCo & 67.91 ± 1.63* & 3.41 ± 2.28 & 0.14 ± 0.02 & 80.15 ± 1.86* & 2.03 ± 1.30 & 0.18 ± 0.03*\\
- & ECCCo (no CP) & 40.72 ± 0.46\hphantom{*}\hphantom{*} & 1.70 ± 4.33\hphantom{*}\hphantom{*} & 41.33 ± 0.30\hphantom{*}\hphantom{*} & 1.72 ± 2.09\hphantom{*}\hphantom{*} & 2.51 ± 0.34\hphantom{*}\hphantom{*} & 4.74 ± 0.79\hphantom{*}\hphantom{*}\\
+ & ECCCo+ & 82.72 ± 1.12* & 2.71 ± 2.32 & 0.17 ± 0.12 & 98.25 ± 0.57 & 1.64 ± 1.01 & 0.20 ± 0.04\\
- & ECCCo (no EBM) & 49.88 ± 0.32\hphantom{*}\hphantom{*} & 1.69 ± 4.33\hphantom{*}\hphantom{*} & 38.30 ± 0.28\hphantom{*}\hphantom{*} & 1.72 ± 2.08\hphantom{*}\hphantom{*} & 3.11 ± 0.27\hphantom{*}\hphantom{*} & 4.76 ± 0.80\hphantom{*}\hphantom{*}\\
+ & ECCCo (no CP) & 88.72 ± 2.28* & 3.40 ± 2.28 & 0.14 ± 0.03 & 82.52 ± 1.18* & 2.02 ± 1.30 & 0.18 ± 0.03\\
- & ECCCo-Δ & 108.32 ± 2.14\hphantom{*}\hphantom{*} & 1.68 ± 4.30\hphantom{*}\hphantom{*} & 93.36 ± 0.83\hphantom{*}\hphantom{*} & 1.72 ± 2.13\hphantom{*}\hphantom{*} & 7.32 ± 1.26\hphantom{*}\hphantom{*} & 4.76 ± 0.80\hphantom{*}\hphantom{*}\\
+ & ECCCo (no EBM) & 75.47 ± 1.60* & 0.98 ± 0.32 & 0.15 ± 0.03 & 92.86 ± 1.05 & 1.00 ± 0.77 & 0.20 ± 0.04\\
- & ECCCo-Δ (no CP) & 103.50 ± 1.56\hphantom{*}\hphantom{*} & 1.67 ± 4.30\hphantom{*}\hphantom{*} & 98.37 ± 0.89\hphantom{*}\hphantom{*} & 1.72 ± 2.12\hphantom{*}\hphantom{*} & 6.87 ± 1.27\hphantom{*}\hphantom{*} & 4.76 ± 0.80\hphantom{*}\hphantom{*}\\
+ & REVISE & 98.98 ± 0.23* & 0.64 ± 0.19* & 0.21 ± 0.15 & 118.36 ± 1.68 & 0.71 ± 0.37 & 0.28 ± 0.08\\
- & ECCCo-Δ (no EBM) & 80.25 ± 1.30\hphantom{*}\hphantom{*} & 1.69 ± 4.33\hphantom{*}\hphantom{*} & 114.56 ± 1.15\hphantom{*}\hphantom{*} & 1.72 ± 2.08\hphantom{*}\hphantom{*} & 7.61 ± 1.51\hphantom{*}\hphantom{*} & 4.76 ± 0.80\hphantom{*}\hphantom{*}\\
+ & Schut & 87.66 ± 2.05* & 1.02 ± 0.31 & 0.13 ± 0.00* & 114.37 ± 1.21 & 1.32 ± 0.72 & 0.16 ± 0.00*\\
- & REVISE & 93.47 ± 0.76\hphantom{*}\hphantom{*} & 1.47 ± 3.73\hphantom{*}\hphantom{*} & 96.92 ± 2.21\hphantom{*}\hphantom{*} & 0.94 ± 1.31\hphantom{*}\hphantom{*} & 7.35 ± 1.29\hphantom{*}\hphantom{*} & 3.55 ± 0.21\hphantom{*}\hphantom{*}\\
+\multirow{-7}{*}{\raggedright\arraybackslash MLP} & Wachter & 114.38 ± 2.14 & 0.98 ± 0.32 & 0.16 ± 0.04 & 84.37 ± 0.99 & 0.99 ± 0.77 & 0.21 ± 0.05\\
+\cmidrule{1-8}
+ & ECCCo & 65.20 ± 7.42* & 0.73 ± 0.26* & 0.11 ± 0.02* & 76.30 ± 4.15* & 0.78 ± 0.20 & 0.24 ± 0.20\\
- & Schut & 85.92 ± 1.95\hphantom{*}\hphantom{*} & 1.85 ± 4.30\hphantom{*}\hphantom{*} & 120.89 ± 1.23\hphantom{*}\hphantom{*} & 1.54 ± 2.04\hphantom{*}\hphantom{*} & 7.08 ± 1.42\hphantom{*}\hphantom{*} & 4.85 ± 0.79\hphantom{*}\hphantom{*}\\
+ & ECCCo+ & 81.01 ± 3.33* & 0.63 ± 0.20* & 0.12 ± 0.02* & 96.72 ± 7.19 & 0.68 ± 0.12* & 0.21 ± 0.19\\
-\multirow{-9}{*}{\raggedright\arraybackslash MLP} & Wachter & 99.17 ± 2.21\hphantom{*}\hphantom{*} & 1.69 ± 4.33\hphantom{*}\hphantom{*} & 91.30 ± 1.54\hphantom{*}\hphantom{*} & 1.72 ± 2.07\hphantom{*}\hphantom{*} & 7.88 ± 0.73\hphantom{*}\hphantom{*} & 4.76 ± 0.80\hphantom{*}\hphantom{*}\\
+ & ECCCo (no CP) & 84.79 ± 0.49* & 0.73 ± 0.27* & 0.12 ± 0.03* & 82.18 ± 4.59 & 0.78 ± 0.19 & 0.22 ± 0.22\\
+
+ & ECCCo (no EBM) & 77.03 ± 5.57* & 1.07 ± 0.51 & 0.48 ± 0.28 & 87.72 ± 1.01 & 0.87 ± 0.20 & 0.13 ± 0.02\\
+
+ & REVISE & 94.52 ± 0.61* & 0.54 ± 0.12* & 0.21 ± 0.12* & 111.55 ± 5.38 & 0.66 ± 0.11* & 0.23 ± 0.08\\
+
+ & Schut & 85.81 ± 4.65* & 1.21 ± 0.39 & 0.73 ± 0.16 & 110.80 ± 5.99 & 1.25 ± 0.22 & 0.10 ± 0.01*\\
+
+\multirow{-7}{*}{\raggedright\arraybackslash JEM Ensemble} & Wachter & 107.85 ± 2.52 & 1.12 ± 0.48 & 0.68 ± 0.22 & 82.68 ± 3.58 & 0.86 ± 0.20 & 0.13 ± 0.02\\
\bottomrule
-\end{tabu}}
-\end{table}
+\end{tabular}}
+\end{table*}
diff --git a/paper/contents/table-vision.tex b/paper/contents/table-vision.tex
new file mode 100644
index 0000000000000000000000000000000000000000..e7e7559cc270da106727455815cb7a2a0929c33a
--- /dev/null
+++ b/paper/contents/table-vision.tex
@@ -0,0 +1,33 @@
+\begin{table}
+
+\caption{Results for vision dataset. Formatting details are the same as in Table~\ref{tab:results-tabular}. \label{tab:results-vision} \newline}
+\centering
+\resizebox{\linewidth}{!}{
+\begin{tabular}[t]{llcc}
+\toprule
+\multicolumn{2}{c}{ } & \multicolumn{2}{c}{MNIST} \\
+\cmidrule(l{3pt}r{3pt}){3-4}
+Model & Generator & Unfaithfulness ↓ & Implausibility ↓\\
+\midrule
+ & ECCCo & 0.22 ± 0.01* & 0.42 ± 0.02\\
+
+ & ECCCo+ & 0.23 ± 0.01* & 0.32 ± 0.02*\\
+
+ & REVISE & 0.24 ± 0.01 & 0.30 ± 0.03*\\
+
+ & Schut & 0.25 ± 0.01 & 0.34 ± 0.03*\\
+
+\multirow{-5}{*}{\raggedright\arraybackslash MLP} & Wachter & 0.24 ± 0.01 & 0.37 ± 0.04\\
+\cmidrule{1-4}
+ & ECCCo & 0.24 ± 0.01 & 0.39 ± 0.03\\
+
+ & ECCCo+ & 0.24 ± 0.01 & 0.33 ± 0.02\\
+
+ & REVISE & 0.25 ± 0.01 & 0.30 ± 0.03*\\
+
+ & Schut & 0.25 ± 0.01 & 0.34 ± 0.03\\
+
+\multirow{-5}{*}{\raggedright\arraybackslash LeNet-5} & Wachter & 0.25 ± 0.01 & 0.35 ± 0.03\\
+\bottomrule
+\end{tabular}}
+\end{table}
diff --git a/paper/contents/table_ebm_params.tex b/paper/contents/table_ebm_params.tex
index e7fe1e0907384a1c0cf18ccdcef83720d6b91d45..1377cd7c8a97c25584ec9b18d7af42e49156b02f 100644
--- a/paper/contents/table_ebm_params.tex
+++ b/paper/contents/table_ebm_params.tex
@@ -7,11 +7,14 @@
\toprule
Dataset & SGLD Steps & Batch Size & $\lambda$\\
\midrule
-Linearly Separable & 30 & 50 & 0.10\\
+Linearly Separable & 50 & 50 & 0.10\\
Moons & 30 & 10 & 0.10\\
-Circles & 20 & 100 & 0.01\\
-MNIST & 25 & 10 & 0.01\\
+Circles & 30 & 50 & 0.01\\
+California Housing & 30 & 10 & 0.10\\
GMSC & 30 & 10 & 0.10\\
+German Credit & 30 & 10 & 0.10\\
+MNIST & 25 & 10 & 0.01\\
+Fashion MNIST & 25 & 10 & 0.01\\
\bottomrule
\end{tabular}
\end{table}
diff --git a/paper/contents/table_gen_params.tex b/paper/contents/table_gen_params.tex
index 84b89401bdaebddd0a0c92778fe91d5cc0122d2b..94e6b2ac0bb6087d2748619cc15fa2db0b5b4d75 100644
--- a/paper/contents/table_gen_params.tex
+++ b/paper/contents/table_gen_params.tex
@@ -7,11 +7,14 @@
\toprule
Dataset & $\eta$ & $\lambda_1$ & $\lambda_2$ & $\lambda_3$\\
\midrule
-Linearly Separable & 0.01 & 0.25 & 0.75 & 0.75\\
-Moons & 0.05 & 0.25 & 0.75 & 0.75\\
-Circles & 0.01 & 0.25 & 0.75 & 0.75\\
-MNIST & 0.10 & 0.10 & 0.25 & 0.25\\
-GMSC & 0.05 & 0.10 & 0.50 & 0.50\\
+Linearly Separable & 0.01 & 0.10 & 0.10 & 0.05\\
+Moons & 0.01 & 0.10 & 0.10 & 0.50\\
+Circles & 0.05 & 0.10 & 0.10 & 0.05\\
+California Housing & 0.05 & 0.10 & 0.10 & 0.10\\
+GMSC & 0.05 & 0.10 & 0.10 & 0.10\\
+German Credit & 0.05 & 0.20 & 0.20 & 0.20\\
+MNIST & 0.10 & 0.01 & 0.25 & 0.25\\
+Fashion MNIST & 0.10 & 0.01 & 0.25 & 0.25\\
\bottomrule
\end{tabular}
\end{table}
diff --git a/paper/contents/table_params.tex b/paper/contents/table_params.tex
index d0ccc5a3ef1f60a4ff3f5d39f629e41ae4d3d46e..ee68e0ebdcb46a6f74d6ac5a1e142be40e497502 100644
--- a/paper/contents/table_params.tex
+++ b/paper/contents/table_params.tex
@@ -11,9 +11,12 @@ Dataset & Sample Size & Hidden Units & Hidden Layers & Activation & Ensemble Siz
\midrule
Linearly Separable & 1000 & 16 & 3 & swish & 5 & 100 & 100\\
Moons & 2500 & 32 & 3 & relu & 5 & 500 & 128\\
-Circles & 1000 & 32 & 3 & swish & 5 & 100 & 100\\
-MNIST & 10000 & 128 & 1 & swish & 5 & 100 & 128\\
-GMSC & 13370 & 128 & 2 & swish & 5 & 100 & 250\\
+Circles & 1000 & 32 & 1 & swish & 5 & 100 & 100\\
+California Housing & 16500 & 32 & 3 & relu & 5 & 100 & 128\\
+GMSC & 13370 & 32 & 3 & relu & 5 & 100 & 128\\
+German Credit & 800 & 32 & 3 & relu & 5 & 100 & 80\\
+MNIST & 10000 & 32 & 1 & relu & 5 & 100 & 128\\
+Fashion MNIST & 10000 & 32 & 2 & relu & 5 & 100 & 128\\
\bottomrule
\end{tabular}}
\end{table}
diff --git a/paper/contents/table_perf.tex b/paper/contents/table_perf.tex
index 737a98e599580f7d0a38c2c1ec377b9fbed577a8..4185b00e664bb8f3d11b14b8bf67218ad8d9676d 100644
--- a/paper/contents/table_perf.tex
+++ b/paper/contents/table_perf.tex
@@ -9,33 +9,73 @@
\cmidrule(l{3pt}r{3pt}){3-5}
Dataset & Model & Accuracy & Precision & F1-Score\\
\midrule
- & JEM & 0.99 & 0.99 & 0.99\\
+ & JEM & 0.98 & 0.98 & 0.98\\
+
+ & JEM Ensemble & 0.99 & 0.99 & 0.99\\
+
+ & MLP & 0.99 & 0.99 & 0.99\\
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Linearly Separable} & MLP & 0.99 & 0.99 & 0.99\\
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash Linearly Separable} & MLP Ensemble & 0.99 & 0.99 & 0.99\\
\cmidrule{1-5}
& JEM & 1.00 & 1.00 & 1.00\\
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Moons} & MLP & 1.00 & 1.00 & 1.00\\
+ & JEM Ensemble & 1.00 & 1.00 & 1.00\\
+
+ & MLP & 1.00 & 1.00 & 1.00\\
+
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash Moons} & MLP Ensemble & 1.00 & 1.00 & 1.00\\
\cmidrule{1-5}
- & JEM & 0.98 & 0.98 & 0.98\\
+ & JEM & 1.00 & 1.00 & 1.00\\
+
+ & JEM Ensemble & 1.00 & 1.00 & 1.00\\
+
+ & MLP & 1.00 & 1.00 & 1.00\\
-\multirow[t]{-2}{*}{\raggedleft\arraybackslash Circles} & MLP & 1.00 & 1.00 & 1.00\\
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash Circles} & MLP Ensemble & 1.00 & 1.00 & 1.00\\
\cmidrule{1-5}
- & JEM & 0.83 & 0.84 & 0.83\\
+ & JEM & 0.87 & 0.87 & 0.87\\
- & JEM Ensemble & 0.90 & 0.90 & 0.89\\
+ & JEM Ensemble & 0.87 & 0.87 & 0.87\\
+
+ & MLP & 0.89 & 0.89 & 0.89\\
+
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash California Housing} & MLP Ensemble & 0.89 & 0.89 & 0.89\\
+\cmidrule{1-5}
+ & JEM & 0.75 & 0.76 & 0.74\\
+
+ & JEM Ensemble & 0.74 & 0.75 & 0.74\\
+
+ & MLP & 0.74 & 0.75 & 0.74\\
+
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash GMSC} & MLP Ensemble & 0.74 & 0.74 & 0.74\\
+\cmidrule{1-5}
+ & JEM & 0.54 & 0.60 & 0.47\\
+
+ & JEM Ensemble & 0.55 & 0.68 & 0.46\\
+
+ & MLP & 0.54 & 0.76 & 0.42\\
+
+\multirow[t]{-4}{*}{\raggedleft\arraybackslash German Credit} & MLP Ensemble & 0.51 & 0.75 & 0.36\\
+\cmidrule{1-5}
+ & JEM & 0.84 & 0.85 & 0.84\\
+
+ & JEM Ensemble & 0.90 & 0.90 & 0.90\\
+
+ & LeNet-5 & 0.98 & 0.98 & 0.98\\
& MLP & 0.95 & 0.95 & 0.95\\
-\multirow[t]{-4}{*}{\raggedleft\arraybackslash MNIST} & MLP Ensemble & 0.95 & 0.95 & 0.95\\
+\multirow[t]{-5}{*}{\raggedleft\arraybackslash MNIST} & MLP Ensemble & 0.95 & 0.95 & 0.95\\
\cmidrule{1-5}
- & JEM & 0.73 & 0.75 & 0.73\\
+ & JEM & 0.62 & 0.70 & 0.62\\
+
+ & JEM Ensemble & 0.78 & 0.78 & 0.78\\
- & JEM Ensemble & 0.73 & 0.75 & 0.73\\
+ & LeNet-5 & 0.83 & 0.84 & 0.82\\
- & MLP & 0.75 & 0.75 & 0.75\\
+ & MLP & 0.82 & 0.83 & 0.82\\
-\multirow[t]{-4}{*}{\raggedleft\arraybackslash GMSC} & MLP Ensemble & 0.75 & 0.75 & 0.75\\
+\multirow[t]{-5}{*}{\raggedleft\arraybackslash Fashion MNIST} & MLP Ensemble & 0.84 & 0.84 & 0.84\\
\bottomrule
\end{tabular}
\end{table}
diff --git a/paper/neurips_2023.sty b/paper/neurips/neurips_2023.sty
similarity index 100%
rename from paper/neurips_2023.sty
rename to paper/neurips/neurips_2023.sty
diff --git a/paper/paper.pdf b/paper/neurips/paper.pdf
similarity index 57%
rename from paper/paper.pdf
rename to paper/neurips/paper.pdf
index 901f581eefddd488d2e0de87932cf9cc3f8de8f8..dfee8692887d231ea444e982fadc950d6882cc30 100644
Binary files a/paper/paper.pdf and b/paper/neurips/paper.pdf differ
diff --git a/paper/neurips/paper.tex b/paper/neurips/paper.tex
new file mode 100644
index 0000000000000000000000000000000000000000..958928cff8e8e376e5d2c98a5c3943f9a3197db5
--- /dev/null
+++ b/paper/neurips/paper.tex
@@ -0,0 +1,105 @@
+\documentclass{article}
+
+
+% if you need to pass options to natbib, use, e.g.:
+% \PassOptionsToPackage{numbers, compress}{natbib}
+% before loading neurips_2023
+
+
+% ready for submission
+\usepackage{neurips_2023}
+
+
+% to compile a preprint version, e.g., for submission to arXiv, add add the
+% [preprint] option:
+% \usepackage[preprint]{neurips_2023}
+
+
+% to compile a camera-ready version, add the [final] option, e.g.:
+% \usepackage[final]{neurips_2023}
+
+
+% to avoid loading the natbib package, add option nonatbib:
+% \usepackage[nonatbib]{neurips_2023}
+
+
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc} % use 8-bit T1 fonts
+\usepackage{hyperref} % hyperlinks
+\usepackage{url} % simple URL typesetting
+\usepackage{booktabs} % professional-quality tables
+\usepackage{amsfonts} % blackboard math symbols
+\usepackage{nicefrac} % compact symbols for 1/2, etc.
+\usepackage{microtype} % microtypography
+\usepackage{xcolor} % colors
+
+\usepackage{amsmath}
+\usepackage{amsthm}
+\usepackage{caption}
+\usepackage{graphicx}
+\usepackage{algorithm}
+\usepackage{algpseudocode}
+\usepackage{import}
+\usepackage{booktabs}
+\usepackage{longtable}
+\usepackage{array}
+\usepackage{multirow}
+
+% Bibliography
+\bibliographystyle{unsrtnat}
+\setcitestyle{numbers,square,comma}
+
+% Numbered Environments:
+\newtheorem{definition}{Definition}[section]
+\newtheorem{question}{Research Question}[section]
+
+\renewcommand{\algorithmicrequire}{\textbf{Input:}}
+\renewcommand{\algorithmicensure}{\textbf{Output:}}
+
+
+\title{ECCCos from the Black Box:\\
+Faithful Explanations through\\
+Energy-Constrained Conformal Counterfactuals}
+
+
+% The \author macro works with any number of authors. There are two commands
+% used to separate the names and addresses of multiple authors: \And and \AND.
+%
+% Using \And between authors leaves it to LaTeX to determine where to break the
+% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
+% authors names on the first line, and the last on the second line, try using
+% \AND instead of \And before the third author name.
+
+
+\author{%
+ Anonymous Author\thanks{See also: } \\
+ Faculty \\
+ University \\
+ Address \\
+ \texttt{email} \\
+ \And
+ Anonymous Author\thanks{See also: } \\
+ Faculty \\
+ University \\
+ Address \\
+ \texttt{email} \\
+}
+
+
+\begin{document}
+
+% Body of the paper
+\import{../}{body.tex}
+
+\begin{ack}
+
+Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
+collaboration.
+
+\end{ack}
+
+\bibliography{../bib}
+
+\import{../}{appendix.tex}
+
+\end{document}
\ No newline at end of file
diff --git a/paper/paper.tex b/paper/paper.tex
deleted file mode 100644
index 58c9610bf326e89cf055b136131457967c80834f..0000000000000000000000000000000000000000
--- a/paper/paper.tex
+++ /dev/null
@@ -1,491 +0,0 @@
-\documentclass{article}
-
-
-% if you need to pass options to natbib, use, e.g.:
-% \PassOptionsToPackage{numbers, compress}{natbib}
-% before loading neurips_2023
-
-
-% ready for submission
-\usepackage{neurips_2023}
-
-
-% to compile a preprint version, e.g., for submission to arXiv, add add the
-% [preprint] option:
-% \usepackage[preprint]{neurips_2023}
-
-
-% to compile a camera-ready version, add the [final] option, e.g.:
-% \usepackage[final]{neurips_2023}
-
-
-% to avoid loading the natbib package, add option nonatbib:
-% \usepackage[nonatbib]{neurips_2023}
-
-
-\usepackage[utf8]{inputenc} % allow utf-8 input
-\usepackage[T1]{fontenc} % use 8-bit T1 fonts
-\usepackage{hyperref} % hyperlinks
-\usepackage{url} % simple URL typesetting
-\usepackage{booktabs} % professional-quality tables
-\usepackage{amsfonts} % blackboard math symbols
-\usepackage{nicefrac} % compact symbols for 1/2, etc.
-\usepackage{microtype} % microtypography
-\usepackage{xcolor} % colors
-
-\usepackage{amsmath}
-\usepackage{amsthm}
-\usepackage{caption}
-\usepackage{graphicx}
-\usepackage{algorithm}
-\usepackage{algpseudocode}
-\usepackage{import}
-\usepackage{booktabs}
-\usepackage{longtable}
-\usepackage{array}
-\usepackage{multirow}
-
-% Bibliography
-\bibliographystyle{unsrtnat}
-\setcitestyle{numbers,square,comma}
-
-% Numbered Environments:
-\newtheorem{definition}{Definition}[section]
-\newtheorem{question}{Research Question}[section]
-
-
-\title{ECCCos from the Black Box:\\
-Faithful Explanations through\\
-Energy-Constrained Conformal Counterfactuals}
-
-
-% The \author macro works with any number of authors. There are two commands
-% used to separate the names and addresses of multiple authors: \And and \AND.
-%
-% Using \And between authors leaves it to LaTeX to determine where to break the
-% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
-% authors names on the first line, and the last on the second line, try using
-% \AND instead of \And before the third author name.
-
-
-\author{%
- Anonymous Author\thanks{See also: } \\
- Faculty \\
- University \\
- Address \\
- \texttt{email} \\
- \And
- Anonymous Author\thanks{See also: } \\
- Faculty \\
- University \\
- Address \\
- \texttt{email} \\
-}
-
-
-\begin{document}
-
-
-\maketitle
-
-
-\begin{abstract}
- Counterfactual Explanations offer an intuitive and straightforward way to explain black-box models and offer Algorithmic Recourse to individuals. To address the need for plausible explanations, existing work has primarily relied on surrogate models to learn how the input data is distributed. This effectively reallocates the task of learning realistic explanations for the data from the model itself to the surrogate. Consequently, the generated explanations may seem plausible to humans but need not necessarily describe the behaviour of the black-box model faithfully. We formalise this notion of faithfulness through the introduction of a tailored evaluation metric and propose a novel algorithmic framework for generating \textbf{E}nergy-\textbf{C}onstrained \textbf{C}onformal \textbf{Co}unterfactuals (ECCCos) that are only as plausible as the model permits. Through extensive empirical studies, we demonstrate that ECCCos reconcile the need for faithfulness and plausibility. In particular, we show that for models with gradient access, it is possible to achieve state-of-the-art performance without the need for surrogate models. To do so, our framework relies solely on properties defining the black-box model itself by leveraging recent advances in Energy-Based Modelling and Conformal Prediction. To our knowledge, this is the first venture in this direction for generating faithful Counterfactual Explanations. Thus, we anticipate that ECCCos can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
-\end{abstract}
-
-\section{Introduction}\label{intro}
-
-Counterfactual Explanations (CE) provide a powerful, flexible and intuitive way to not only explain black-box models but also help affected individuals through the means of Algorithmic Recourse. Instead of opening the Black Box, CE works under the premise of strategically perturbing model inputs to understand model behaviour~\citep{wachter2017counterfactual}. Intuitively speaking, we generate explanations in this context by asking what-if questions of the following nature: `Our credit risk model currently predicts that this individual is not credit-worthy. What if they reduced their monthly expenditures by 10\%?'
-
-This is typically implemented by defining a target outcome $\mathbf{y}^+ \in \mathcal{Y}$ for some individual $\mathbf{x} \in \mathcal{X}=\mathbb{R}^D$ described by $D$ attributes, for which the model $M_{\theta}:\mathcal{X}\mapsto\mathcal{Y}$ initially predicts a different outcome: $M_{\theta}(\mathbf{x})\ne \mathbf{y}^+$. Counterfactuals are then searched by minimizing a loss function that compares the predicted model output to the target outcome: $\text{yloss}(M_{\theta}(\mathbf{x}),\mathbf{y}^+)$. Since CE work directly with the black-box model, valid counterfactuals always have full local fidelity by construction where fidelity is defined as the degree to which explanations approximate the predictions of a black-box model~\citep{mothilal2020explaining,molnar2020interpretable}.
-
-In situations where full fidelity is a requirement, CE offer a more appropriate solution to Explainable Artificial Intelligence (XAI) than other popular approaches like LIME~\citep{ribeiro2016why} and SHAP~\citep{lundberg2017unified}, which involve local surrogate models. But even full fidelity is not a sufficient condition for ensuring that an explanation faithfully describes the behaviour of a model. That is because multiple very distinct explanations can all lead to the same model prediction, especially when dealing with heavily parameterized models like deep neural networks, which are typically underspecified by the data~\citep{wilson2020case}.
-
-In the context of CE, the idea that no two explanations are the same arises almost naturally. A key focus in the literature has therefore been to identify those explanations and algorithmic recourses that are most appropriate based on a myriad of desiderata such as sparsity, actionability and plausibility. In this work, we draw closer attention to model faithfulness rather than fidelity as a desideratum for counterfactuals. Our key contributions are as follows:
-
-\begin{itemize}
- \item We show that fidelity is an insufficient evaluation metric for counterfactuals (Section~\ref{fidelity}) and propose a definition of faithfulness that gives rise to more suitable metrics (Section~\ref{faithfulness}).
- \item We introduce a novel algorithmic approach for generating Energy-Constrained Conformal Counterfactuals (ECCCos) in Section~\ref{meth}.
- \item We provide extensive empirical evidence demonstrating that ECCCos faithfully explain model behaviour and attain plausibility only when appropriate (Section~\ref{emp}).
-\end{itemize}
-
-To our knowledge, this is the first venture in this direction for generating faithful counterfactuals. Thus, we anticipate that ECCCos can serve as a baseline for future research. We believe that our work opens avenues for researchers and practitioners seeking tools to better distinguish trustworthy from unreliable models.
-
-\section{Background}\label{background}
-
-While CE can also be generated for arbitrary regression models~\citep{spooner2021counterfactual}, existing work has primarily focused on classification problems. Let $\mathcal{Y}=(0,1)^K$ denote the one-hot-encoded output domain with $K$ classes. Then most counterfactual generators rely on gradient descent to optimize different flavours of the following counterfactual search objective:
-
-\begin{equation} \label{eq:general}
-\begin{aligned}
-\mathbf{Z}^\prime &= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^L} \left\{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda {\text{cost}(f(\mathbf{Z}^\prime)) } \right\}
-\end{aligned}
-\end{equation}
-
-Here $\text{yloss}(\cdot)$ denotes the primary loss function, $f(\cdot)$ is a function that maps from the counterfactual state space to the feature space and $\text{cost}(\cdot)$ is either a single penalty or a collection of penalties that are used to impose constraints through regularization. Equation~\ref{eq:general} restates the baseline approach to gradient-based counterfactual search proposed by~\citet{wachter2017counterfactual} in general form as introduced by~\citet{altmeyer2023endogenous}. To explicitly account for the multiplicity of explanations, $\mathbf{Z}^\prime=\{ \mathbf{z}_l\}_L$ denotes an $L$-dimensional array of counterfactual states.
-
-The baseline approach, which we will simply refer to as \textit{Wachter}, searches a single counterfactual directly in the feature space and penalises its distance to the original factual. In this case, $f(\cdot)$ is simply the identity function and $\mathcal{Z}$ corresponds to the feature space itself. Many derivative works of~\citet{wachter2017counterfactual} have proposed new flavours of Equation~\ref{eq:general}, each of them designed to address specific \textit{desiderata} that counterfactuals ought to meet in order to properly serve both AI practitioners and individuals affected by algorithmic decision-making systems. The list of desiderata includes but is not limited to the following: sparsity, proximity~\citep{wachter2017counterfactual}, actionability~\citep{ustun2019actionable}, diversity~\citep{mothilal2020explaining}, plausibility~\citep{joshi2019realistic,poyiadzi2020face,schut2021generating}, robustness~\citep{upadhyay2021robust,pawelczyk2022probabilistically,altmeyer2023endogenous} and causality~\citep{karimi2021algorithmic}. Different counterfactual generators addressing these needs have been extensively surveyed and evaluated in various studies~\citep{verma2020counterfactual,karimi2020survey,pawelczyk2021carla,artelt2021evaluating,guidotti2022counterfactual}.
-
-Perhaps unsurprisingly, the different desiderata are often positively correlated. For example, \citet{artelt2021evaluating} find that plausibility typically also leads to improved robustness. Similarly, plausibility has also been connected to causality in the sense that plausible counterfactuals respect causal relationships~\citep{mahajan2020preserving}. Consequently, the plausibility of counterfactuals has been among the primary concerns for researchers. Achieving plausibility is equivalent to ensuring that the generated counterfactuals comply with the true and unobserved data-generating process (DGP). We define plausibility formally in this work as follows:
-
-\begin{definition}[Plausible Counterfactuals]
- \label{def:plausible}
- Let $\mathcal{X}|\mathbf{y}^+= p(\mathbf{x}|\mathbf{y}^+)$ denote the true conditional distribution of samples in the target class $\mathbf{y}^+$. Then for $\mathbf{x}^{\prime}$ to be considered a plausible counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}|\mathbf{y}^+$.
-\end{definition}
-
-To generate plausible counterfactuals, we need to be able to quantify the DGP: $\mathcal{X}|\mathbf{y}^+$. One straightforward way to do this is to use surrogate models for the task. \citet{joshi2019realistic}, for example, suggest that instead of searching counterfactuals in the feature space $\mathcal{X}$, we can instead traverse a latent embedding $\mathcal{Z}$ (Equation~\ref{eq:general}) that implicitly codifies the DGP. To learn the latent embedding, they propose using a generative model such as a Variational Autoencoder (VAE). Provided the surrogate model is well-specified, their proposed approach called \textit{REVISE} can yield plausible explanations. Others have proposed similar approaches: \citet{dombrowski2021diffeomorphic} traverse the base space of a normalizing flow to solve Equation~\ref{eq:general}; \citet{poyiadzi2020face} use density estimators ($\hat{p}: \mathcal{X} \mapsto [0,1]$) to constrain the counterfactuals to dense regions in the feature space; and, finally, \citet{karimi2021algorithmic} assume knowledge about the structural causal model that generates the data.
-
-A competing approach towards plausibility that is also closely related to this work instead relies on the black-box model itself. \citet{schut2021generating} show that to meet the plausibility objective we need not explicitly model the input distribution. Pointing to the undesirable engineering overhead induced by surrogate models, they propose that we rely on the implicit minimisation of predictive uncertainty instead. Their proposed methodology, which we will refer to as \textit{Schut}, solves Equation~\ref{eq:general} by greedily applying Jacobian-Based Saliency Map Attacks (JSMA) in the feature space with cross-entropy loss and no penalty at all. The authors demonstrate theoretically and empirically that their approach yields counterfactuals for which the model $M_{\theta}$ predicts the target label $\mathbf{y}^+$ with high confidence. Provided the model is well-specified, these counterfactuals are plausible. This idea hinges on the assumption that the black-box model provides well-calibrated predictive uncertainty estimates.
-
-\section{Why Fidelity is not Enough}\label{fidelity}
-
-As discussed in the introduction, any valid counterfactual also has full fidelity by construction: solutions to Equation~\ref{eq:general} are considered valid as soon as the label predicted by the model matches the target class. So while fidelity always applies, counterfactuals that address the various desiderata introduced above can look vastly different from each other.
-
-To demonstrate this with an example, we have trained a simple image classifier $M_{\theta}$ on the well-known \textit{MNIST} dataset~\citep{lecun1998mnist}: a Multi-Layer Perceptron (\textit{MLP}) with above 90 percent test accuracy. No measures have been taken to improve the model's adversarial robustness or its capacity for predictive uncertainty quantification. The far left panel of Figure ~\ref{fig:motiv} shows a random sample drawn from the dataset. The underlying classifier correctly predicts the label `nine' for this image. For the given factual image and model, we have used \textit{Wachter}, \textit{Schut} and \textit{REVISE} to generate one counterfactual each in the target class `seven'. The perturbed images are shown next to the factual image from left to right in Figure ~\ref{fig:motiv}. Captions on top of the individual images indicate the generator along with the predicted probability that the image belongs to the target class. In all three cases that probability is above 90 percent and yet the counterfactuals look very different from each other.
-
-\begin{figure}
- \centering
- \includegraphics[width=0.8\textwidth]{../artifacts/results/images/mnist_motivation.png}
- \caption{Counterfactuals for turning a 9 (nine) into a 7 (seven): original image (left); then from left to right the counterfactuals generated using \textit{Wachter}, \textit{Schut} and \textit{REVISE}.}\label{fig:motiv}
-\end{figure}
-
-Since \textit{Wachter} is only concerned with proximity, the generated counterfactual is almost indistinguishable from the factual. The approach by~\citet{schut2021generating} expects a well-calibrated model that can generate predictive uncertainty estimates. Since this is not the case, the generated counterfactual looks like an adversarial example. Finally, the counterfactual generated by \textit{REVISE} looks much more plausible than the other two. But is it also more faithful to the behaviour of our \textit{MNIST} classifier? That is much less clear because the surrogate used by \textit{REVISE} introduces friction: the generated explanations no longer depend exclusively on the black-box model itself.
-
-So which of the counterfactuals most faithfully explains the behaviour of our image classifier? Fidelity cannot help us to make that judgement, because all of these counterfactuals have full fidelity. Thus, fidelity is an insufficient evaluation metric to assess the faithfulness of CE.
-
-\section{A New Notion of Faithfulness}\label{faithfulness}
-
-Considering the limitations of fidelity as demonstrated in the previous section, analogous to Definition~\ref{def:plausible}, we introduce a new notion of faithfulness in the context of CE:
-
-\begin{definition}[Faithful Counterfactuals]
- \label{def:faithful}
- Let $\mathcal{X}_{\theta}|\mathbf{y}^+ = p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ denote the conditional distribution of $\mathbf{x}$ in the target class $\mathbf{y}^+$, where $\theta$ denotes the parameters of model $M_{\theta}$. Then for $\mathbf{x}^{\prime}$ to be considered a faithful counterfactual, we need: $\mathbf{x}^{\prime} \sim \mathcal{X}_{\theta}|\mathbf{y}^+$.
-\end{definition}
-
-In doing this, we merge in and nuance the concept of plausibility (Definition~\ref{def:plausible}) where the notion of `consistent with the data' becomes `consistent with what the model has learned about the data'.
-
-\subsection{Quantifying the Model's Generative Property}
-
-To assess counterfactuals with respect to Definition~\ref{def:faithful}, we need a way to quantify the posterior conditional distribution $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$. To this end, we draw on recent advances in Energy-Based Modelling (EBM), a subdomain of machine learning that is concerned with generative or hybrid modelling~\citep{grathwohl2020your,du2020implicit}. In particular, note that if we fix $\mathbf{y}$ to our target value $\mathbf{y}^+$, we can conditionally draw from $p_{\theta}(\mathbf{x}|\mathbf{y}^+)$ by randomly initializing $\mathbf{x}_0$ and then using Stochastic Gradient Langevin Dynamics (SGLD) as follows,
-
-\begin{equation}\label{eq:sgld}
- \begin{aligned}
- \mathbf{x}_{j+1} &\leftarrow \mathbf{x}_j - \frac{\epsilon^2}{2} \mathcal{E}(\mathbf{x}_j|\mathbf{y}^+) + \epsilon \mathbf{r}_j, && j=1,...,J
- \end{aligned}
-\end{equation}
-
-where $\mathbf{r}_j \sim \mathcal{N}(\mathbf{0},\mathbf{I})$ is the stochastic term and the step-size $\epsilon$ is typically polynomially decayed~\citep{welling2011bayesian}. The term $\mathcal{E}(\mathbf{x}_j|\mathbf{y}^+)$ denotes the model energy conditioned on the target class label $\mathbf{y}^+$ which we specify as the negative logit corresponding to the target class label $\mathbf{y}^*$. To allow for faster sampling, we follow the common practice of choosing the step-size $\epsilon$ and the standard deviation of $\mathbf{r}_j$ separately. While $\mathbf{x}_J$ is only guaranteed to distribute as $p_{\theta}(\mathbf{x}|\mathbf{y}^*)$ if $\epsilon \rightarrow 0$ and $J \rightarrow \infty$, the bias introduced for a small finite $\epsilon$ is negligible in practice \citep{murphy2023probabilistic,grathwohl2020your}. Appendix~\ref{app:jem} provides additional implementation details for any tasks related to energy-based modelling.
-
-Generating multiple samples using SGLD thus yields an empirical distribution $\hat{\mathbf{X}}_{\theta,\mathbf{y}^+}$ that approximates what the model has learned about the input data. While in the context of EBM, this is usually done during training, we propose to repurpose this approach during inference in order to evaluate and generate faithful model explanations.
-
-\subsection{Evaluating Plausibility and Faithfulness}
-
-The parallels between our definitions of plausibility and faithfulness imply that we can also use similar evaluation metrics in both cases. Since existing work has focused heavily on plausibility, it offers a useful starting point. In particular,~\citet{guidotti2022counterfactual} have proposed an implausibility metric that measures the distance of the counterfactual from its nearest neighbour in the target class. As this distance is reduced, counterfactuals get more plausible under the assumption that the nearest neighbour itself is plausible in the sense of Definition~\ref{def:plausible}. In this work, we use the following adapted implausibility metric,
-
-\begin{equation}\label{eq:impl}
- \begin{aligned}
- \text{impl}(\mathbf{x}^{\prime},\mathbf{X}_{\mathbf{y}^+}) = \frac{1}{\lvert\mathbf{X}_{\mathbf{y}^+}\rvert} \sum_{\mathbf{x} \in \mathbf{X}_{\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
- \end{aligned}
-\end{equation}
-
-where $\mathbf{x}^{\prime}$ denotes the counterfactual and $\mathbf{X}_{\mathbf{y}^+}$ is a subsample of the training data in the target class $\mathbf{y}^+$. By averaging over multiple samples in this manner, we avoid the risk that the nearest neighbour of $\mathbf{x}^{\prime}$ itself is not plausible according to Definition~\ref{def:plausible} (e.g an outlier).
-
-Equation~\ref{eq:impl} gives rise to a similar evaluation metric for unfaithfulness. We merely swap out the subsample of individuals in the target class for a subset $\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}$ of the generated conditional samples:
-
-\begin{equation}\label{eq:faith}
- \begin{aligned}
- \text{unfaith}(\mathbf{x}^{\prime},\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}) = \frac{1}{\lvert \hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}\rvert} \sum_{\mathbf{x} \in \hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}} \text{dist}(\mathbf{x}^{\prime},\mathbf{x})
- \end{aligned}
-\end{equation}
-
-Specifically, we form this subset based on the $n_E$ generated samples with the lowest energy.
-
-\section{Energy-Constrained Conformal Counterfactuals}\label{meth}
-
-In this section, we describe \textit{ECCCo}, our proposed framework for generating Energy-Constrained Conformal Counterfactuals (ECCCos). It is based on the premise that counterfactuals should first and foremost be faithful. Plausibility, as a secondary concern, is then still attainable, but only to the degree that the black-box model itself has learned plausible explanations for the underlying data.
-
-We begin by stating our proposed objective function, which involves tailored loss and penalty functions that we will explain in the following. In particular, we extend Equation~\ref{eq:general} as follows:
-
-\begin{equation} \label{eq:eccco}
- \begin{aligned}
- \mathbf{Z}^\prime= \arg \min_{\mathbf{Z}^\prime \in \mathcal{Z}^M} &\{ {\text{yloss}(M_{\theta}(f(\mathbf{Z}^\prime)),\mathbf{y}^+)}+ \lambda_{1} {\text{dist}(f(\mathbf{Z}^\prime),\mathbf{x}) } \\
- &+ \lambda_2 \text{unfaith}(f(\mathbf{Z}^\prime),\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}) + \lambda_3 \Omega(C_{\theta}(f(\mathbf{Z}^\prime);\alpha)) \}
- \end{aligned}
-\end{equation}
-
-The first penalty term involving $\lambda_1$ induces proximity like in~\citet{wachter2017counterfactual}. Our default choice for $\text{dist}(\cdot)$ is the L1 Norm due to its sparsity-inducing properties. The second penalty term involving $\lambda_2$ induces faithfulness by constraining the energy of the generated counterfactual where $\text{unfaith}(\cdot)$ corresponds to the metric defined in Equation~\ref{eq:faith}. The third and final penalty term involving $\lambda_3$ introduces a new concept: it ensures that the generated counterfactual is associated with low predictive uncertainty. As mentioned above,~\citet{schut2021generating} have shown that plausible counterfactuals can be generated implicitly through predictive uncertainty minimization. Unfortunately, this relies on the assumption that the model itself can provide predictive uncertainty estimates, which may be too restrictive in practice.
-
-To relax this assumption, we leverage recent advances in Conformal Prediction (CP), an approach to predictive uncertainty quantification that has recently gained popularity~\citep{angelopoulos2021gentle,manokhin2022awesome}. Crucially for our intended application, CP is model-agnostic and can be applied during inference without placing any restrictions on model training. Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the training data or a dedicated calibration dataset. Conformal classifiers produce prediction sets for individual inputs that include all output labels that can be reasonably attributed to the input. These sets tend to be larger for inputs that do not conform with the training data and are characterized by high predictive uncertainty.
-
-In order to generate counterfactuals that are associated with low predictive uncertainty, we use a smooth set size penalty introduced by~\citet{stutz2022learning} in the context of conformal training:
-
-\begin{equation}\label{eq:setsize}
- \begin{aligned}
- \Omega(C_{\theta}(\mathbf{x};\alpha))&=\max \left(0, \sum_{\mathbf{y}\in\mathcal{Y}}C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha) - \kappa \right)
- \end{aligned}
-\end{equation}
-
-Here, $\kappa \in \{0,1\}$ is a hyper-parameter and $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha)$ can be interpreted as the probability of label $\mathbf{y}$ being included in the prediction set. In order to compute this penalty for any black-box model we merely need to perform a single calibration pass through a holdout set $\mathcal{D}_{\text{cal}}$. Arguably, data is typically abundant and in most applications, practitioners tend to hold out a test data set anyway. Consequently, CP removes the restriction on the family of predictive models, at the small cost of reserving a subset of the available data for calibration. This particular case of conformal prediction is referred to as Split Conformal Prediction (SCP) as it involves splitting the training data into a proper training dataset and a calibration dataset. In addition to the smooth set size penalty, we have also experimented with the use of a tailored function for $\text{yloss}(\cdot)$ that enforces that only the target label $\mathbf{y}^+$ is included in the prediction set ~\citet{stutz2022learning}. Further details are provided in Appendix~\ref{app:cp}.
-
-\begin{figure}
- \centering
- \includegraphics[width=1.0\textwidth]{../artifacts/results/images/poc_gradient_fields.png}
- \caption{Gradient fields and counterfactual paths for different generators. The objective is to generate a counterfual in the `blue' class for a sample from the `orange' class. Bright yellow stars indicate conditional samples generated through SGLD. The underlying classifier is a Joint Energy Model.}\label{fig:poc}
-\end{figure}
-
-\renewcommand{\algorithmicrequire}{\textbf{Input:}}
-\renewcommand{\algorithmicensure}{\textbf{Output:}}
-
-\begin{algorithm}
- \caption{The \textit{ECCCo} generator}\label{alg:eccco}
- \begin{algorithmic}[1]
- \Require $\mathbf{x}, \mathbf{y}^+, M_{\theta}, f, \Lambda=[\lambda_1,\lambda_2,\lambda_3], \alpha, \mathcal{D}, T, \eta, n_{\mathcal{B}}, n_E$ where $M_{\theta}(\mathbf{x})\neq\mathbf{y}^+$
- \Ensure $\mathbf{x}^\prime$
- \State Initialize $\mathbf{z}^\prime \gets f^{-1}(\mathbf{x})$ \Comment{Map to counterfactual state space.}
- \State Generate $\left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}} \gets p_{\theta}(\mathbf{x}_{\mathbf{y}^+})$ \Comment{Generate $n_{\mathcal{B}}$ samples using SGLD (Equation~\ref{eq:sgld}).}
- \State Store $\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+} \gets \left\{\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}\right\}_{n_{\mathcal{B}}}$ \Comment{Choose $n_E$ lowest-energy samples.}
- \State Run \textit{SCP} for $M_{\theta}$ using $\mathcal{D}$ \Comment{Calibrate model through Split Conformal Prediction.}
- \State Initialize $t \gets 0$
- \While{\textit{not converged} or $t < T$} \Comment{For convergence conditions see Appendix~\ref{app:eccco}.}
- \State $\mathbf{z}^\prime \gets \mathbf{z}^\prime - \eta \nabla_{\mathbf{z}^\prime} \mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ \Comment{Take gradient step of size $\eta$.}
- \State $t \gets t+1$
- \EndWhile
- \State $\mathbf{x}^\prime \gets f(\mathbf{z}^\prime)$ \Comment{Map back to feature space.}
- \end{algorithmic}
-\end{algorithm}
-
-To provide some further intuition about our objective defined in Equation~\ref{eq:eccco}, Figure~\ref{fig:poc} illustrates how the different components affect the counterfactual search for a synthetic dataset. The underlying classifier is a Joint Energy Model (\textit{JEM}) that was trained to predict the output class (`blue' or `orange') and generate class-conditional samples~\citep{grathwohl2020your}. We have used four different generator flavours to produce a counterfactual in the `blue' class for a sample from the `orange' class: \textit{Wachter}, which only uses the first penalty ($\lambda_2=\lambda_3=0$); \textit{ECCCo (no EBM)}, which does not constrain energy ($\lambda_2=0$); \textit{ECCCo (no CP)}, which involves no set size penalty ($\lambda_3=0$); and, finally, \textit{ECCCo}, which involves all penalties defined in Equation~\ref{eq:eccco}. Arrows indicate (negative) gradients with respect to the objective function at different points in the feature space.
-
-While \textit{Wachter} generates a valid counterfactual, it ends up close to the original starting point consistent with its objective. \textit{ECCCo (no EBM)} pushes the counterfactual further into the target domain to minimize predictive uncertainty, but the outcome is still not plausible. The counterfactual produced by \textit{ECCCo (no CP)} is attracted by the generated samples shown in bright yellow. Since the \textit{JEM} has learned the conditional input distribution reasonably well in this case, the counterfactuals are both faithful and plausible. Finally, the outcome for \textit{ECCCo} looks similar, but the additional smooth set size penalty leads to somewhat faster convergence.
-
-Algorithm~\ref{alg:eccco} describes how exactly \textit{ECCCo} works. For the sake of simplicity and without loss of generality, we limit our attention to generating a single counterfactual $\mathbf{x}^\prime=f(\mathbf{z}^\prime)$. The counterfactual state $\mathbf{z}^\prime$ is initialized by passing the factual $\mathbf{x}$ through a simple feature transformer $f^{-1}$. Next, we generate $n_{\mathcal{B}}$ conditional samples $\hat{\mathbf{x}}_{\theta,\mathbf{y}^+}$ using SGLD (Equation~\ref{eq:sgld}) and store the $n_E$ instances with the lowest energy. We then calibrate the model $M_{\theta}$ through Split Conformal Prediction. Finally, we search counterfactuals through gradient descent where $\mathcal{L}(\mathbf{z}^\prime,\mathbf{y}^+,\hat{\mathbf{X}}^{n_E}_{\theta,\mathbf{y}^+}; \Lambda, \alpha)$ denotes our loss function defined in Equation~\ref{eq:eccco}. The search terminates once the convergence criterium is met or the maximum number of iterations $T$ has been exhausted. Note that the choice of convergence criterium has important implications on the final counterfactual which we explain in Appendix~\ref{app:eccco}.
-
-\section{Empirical Analysis}\label{emp}
-
-Our goal in this section is to shed light on the following research questions:
-
-\begin{question}[Faithfulness]\label{rq:faithfulness}
- Are ECCCos more faithful than counterfactuals produced by our benchmark generators?
-\end{question}
-
-\begin{question}[Balancing Objectives]\label{rq:plausibility}
- Compared to our benchmark generators, how do ECCCos balance the two key objectives of faithfulness and plausibility?
-\end{question}
-
-The second question is motivated by the intuition that faithfulness and plausibility should coincide for models that have learned plausible explanations of the data. Next, we first briefly describe our experimental setup before presenting our main results.
-
-\subsection{Experimental Setup}
-
-To assess and benchmark the performance of our proposed generator against the state of the art, we generate multiple counterfactuals for different models and datasets. In particular, we compare \textit{ECCCo} and its variants to the following counterfactual generators that were introduced above: firstly; \textit{Schut}, which works under the premise of minimizing predictive uncertainty; secondly, \textit{REVISE}, which is state-of-the-art with respect to plausibility; and, finally, \textit{Wachter}, which serves as our baseline.
-
-We use both synthetic and real-world datasets from different domains, all of which are publicly available and commonly used to train and benchmark classification algorithms. We synthetically generate a dataset containing two \textit{Linearly Separable} Gaussian clusters ($n=1000$), as well as the well-known \textit{Circles} ($n=1000$) and \textit{Moons} ($n=2500$) data. Since these data are generated by distributions of varying degrees of complexity, they allow us to assess how the generators and our proposed evaluation metrics handle this.
-
-As for real-world data, we follow~\citet{schut2021generating} and use the \textit{MNIST}~\citep{lecun1998mnist} dataset containing images of handwritten digits such as the example shown above in Figure~\ref{fig:motiv}. From the social sciences domain, we include Give Me Some Credit (\textit{GMSC})~\citep{kaggle2011give}: a tabular dataset that has been studied extensively in the literature on Algorithmic Recourse~\citep{pawelczyk2021carla}. It consists of 11 numeric features that can be used to predict the binary outcome variable indicating whether retail borrowers experience financial distress.
-
-For the predictive modelling tasks, we use simple neural networks (\textit{MLP}) and Joint Energy Models (\textit{JEM}). For the more complex real-world datasets we also use ensembling in each case. Both joint-energy modelling and ensembling have been associated with improved generative properties and adversarial robustness~\citep{grathwohl2020your,lakshminarayanan2016simple}, so we expect this to be positively correlated with the plausibility of ECCCos. To account for stochasticity, we generate multiple counterfactuals for each target class, generator, model and dataset. Specifically, we randomly sample $n^{-}$ times from the subset of individuals for which the given model predicts the non-target class $\mathbf{y}^{-}$ given the current target. We set $n^{-}=25$ for all of our synthetic datasets, $n^{-}=10$ for \textit{GMSC} and $n^{-}=5$ for \textit{MNIST}. Full details concerning our parameter choices, training procedures and model performance can be found in Appendix~\ref{app:setup}.
-
-\subsection{Results for Synthetic Data}
-
-Table~\ref{tab:results-synthetic} shows the key results for the synthetic datasets separated by model (first column) and generator (second column). The numerical columns show sample averages and standard deviations of our key evaluation metrics computed across all counterfactuals. We have highlighted the best outcome for each model and metric in bold. To provide some sense of effect sizes, we have added asterisks to indicate that a given value is at least one ($*$) or two ($**$) standard deviations lower than the baseline (\textit{Wachter}).
-
-Starting with the high-level results for our \textit{Linearly Separable} data, we find that \textit{ECCCo} produces the most faithful counterfactuals for both black-box models. This is consistent with our design since \textit{ECCCo} directly enforces faithfulness through regularization. Crucially though, \textit{ECCCo} also produces the most plausible counterfactuals for both models. This dataset is so simple that even the \textit{MLP} has learned plausible explanations of the input data. Zooming in on the granular details for the \textit{Linearly Separable} data, the results for \textit{ECCCo (no CP)} and \textit{ECCCo (no EBM)} indicate that the positive results are dominated by the effect of quantifying and leveraging the model's generative property (EBM). Conformal Prediction alone only leads to marginally improved faithfulness and plausibility.
-
-The findings for the \textit{Moons} dataset are broadly in line with the findings so far: for the \textit{JEM}, \textit{ECCCo} yields substantially more faithful and plausible counterfactuals than all other generators. For the \textit{MLP}, faithfulness is maintained but counterfactuals are not plausible. This high-level pattern is broadly consistent with other more complex datasets and supportive of our narrative, so it is worth highlighting: ECCCos consistently achieve high faithfulness, which---subject to the quality of the model itself---coincides with high plausibility. By comparison, \textit{REVISE} yields the most plausible counterfactuals for the \textit{MLP}, but it does so at the cost of faithfulness. We also observe that the best results for \textit{ECCCo} are achieved when using both penalties. Once again though, the generative component (EBM) has a stronger impact on the positive results for the \textit{JEM}.
-
-For the \textit{Circles} data, it appears that \textit{REVISE} performs well, but we note that it generates valid counterfactuals only half of the time (see Appendix~\ref{app:results} for a complete overview including additional common evaluation metrics). The underlying VAE with default parameters has not adequately learned the data-generating process. Of course, it is possible to improve generative performance through hyperparameter tuning but this example serves to illustrate that \textit{REVISE} depends on the quality of its surrogate. Independent of the outcome for \textit{REVISE}, however, the results do not seem to indicate that \textit{ECCCo} substantially improves faithfulness and plausibility for the \textit{Circles} data. We think this points to a limitation of our evaluation metrics rather than \textit{ECCCo} itself: computing average distances fails to account for the `wraparound' effect associated with circular data~\citep{gill2010circular}.
-
-\import{contents/}{table-synthetic.tex}
-
-\subsection{Results for Real-World Data}
-
-The results for our real-world datasets are shown in Table~\ref{tab:results-real-world}. Once again the findings indicate that the plausibility of ECCCos is positively correlated with the capacity of the black-box model to distinguish plausible from implausible inputs. The case is very clear for \textit{MNIST}: ECCCos are consistently more faithful than the counterfactuals produced by our benchmark generators and their plausibility gradually improves through ensembling and joint-energy modelling. Interestingly, faithfulness also gradually improves for \textit{REVISE}. This indicates that as our models improve, their generative capacity approaches that of the surrogate VAE used by \textit{REVISE}. The VAE still outperforms our classifiers in this regard, as evident from the fact that \textit{ECCCo} never quite reaches the same level of plausibility as \textit{REVISE}. With reference to Appendix~\ref{app:results} we note that the results for \textit{Schut} need to be discounted as it rarely produces valid counterfactuals for \textit{MNIST}. Relatedly, we find that \textit{ECCCo} is the only generator that consistently achieves full validity. Finally, it is worth noting that \textit{ECCCo} produces counterfactual images with the lowest average predictive uncertainty for all models.
-
-For the tabular credit dataset (\textit{GMSC}) it is inherently challenging to use deep neural networks in order to achieve good discriminative performance~\citep{borisov2021deep,grinsztajn2022why} and generative performance~\citep{liu2023goggle}, respectively. In order to achieve high plausibility, \textit{ECCCo} effectively requires classifiers to achieve good performance for both tasks. Since this is a challenging task even for Joint Energy Models, it is not surprising to find that even though \textit{ECCCo} once again achieves state-of-the-art faithfulness, it is outperformed by \textit{REVISE} and \textit{Schut} with respect to plausibility.
-
-\subsection{Key Takeways}
-
-To conclude this section, we summarize our findings with reference to the opening questions. The results clearly demonstrate that \textit{ECCCo} consistently achieves state-of-the-art faithfulness, as it was designed to do (Research Question~\ref{rq:faithfulness}). A related important finding is that \textit{ECCCo} yields highly plausible explanations provided that they faithfully describe model behaviour (Research Question~\ref{rq:plausibility}). \textit{ECCCo} achieves this result primarily by leveraging the model's generative property.
-
-\import{contents/}{table-real-world.tex}
-
-\section{Limitations}
-
-Even though we have taken considerable measures to study our proposed methodology carefully, limitations can still be identified. In particular, we have found that the performance of \textit{ECCCo} is sensitive to hyperparameter choices. In order to achieve faithfulness, we generally had to penalise the distance from generated samples slightly more than the distance from factual values.
-
-Conversely, we have not found that strongly penalising prediction set sizes had any discernable effect. Our results indicate that CP alone is often not sufficient to achieve faithfulness and plausibility, although we acknowledge that this needs to be investigated more thoroughly through future work.
-
-While our approach is readily applicable to models with gradient access like deep neural networks, more work is needed to generalise it to other machine learning models such as decision trees. Relatedly, common challenges associated with Energy-Based Modelling including sensitivity to scale, training instabilities and sensitivity to hyperparameters also apply to \textit{ECCCo}.
-
-\section{Conclusion}
-
-This work leverages recent advances in Energy-Based Modelling and Conformal Prediction in the context of Explainable Artificial Intelligence. We have proposed a new way to generate counterfactuals that are maximally faithful to the black-box model they aim to explain. Our proposed generator, \textit{ECCCo}, produces plausible counterfactuals if and only if the black-box model itself has learned realistic explanations for the data, which we have demonstrated through rigorous empirical analysis. This should enable researchers and practitioners to use counterfactuals in order to discern trustworthy models from unreliable ones. While the scope of this work limits its generalizability, we believe that \textit{ECCCo} offers a solid baseline for future work on faithful Counterfactual Explanations.
-
-\begin{ack}
-
-Some of the members of TU Delft were partially funded by ICAI AI for Fintech Research, an ING — TU Delft
-collaboration.
-
-\end{ack}
-
-\bibliography{bib}
-
-\pagebreak
-
-\appendix
-\section*{Appendices}
-\renewcommand{\thesubsection}{\Alph{subsection}}
-
-The following appendices provide additional details that are relevant to the paper. Appendices~\ref{app:jem} and~\ref{app:cp} explain any tasks related to Energy-Based Modelling and Predictive Uncertainty Quantification through Conformal Prediction, respectively. Appendix~\ref{app:eccco} provides additional technical and implementation details about our proposed generator, \textit{ECCCo}, including references to our open-sourced code base. A complete overview of our experimental setup detailing our parameter choices, training procedures and initial black-box model performance can be found in Appendix~\ref{app:setup}. Finally, Appendix~\ref{app:results} reports all of our experimental results in more detail.
-
-\subsection{Energy-Based Modelling}\label{app:jem}
-
-Since we were not able to identify any existing open-source software for Energy-Based Modelling that would be flexible enough to cater to our needs, we have developed a \texttt{Julia} package from scratch. The package has been open-sourced, but to avoid compromising the double-blind review process, we refrain from providing more information at this stage. In our development we have heavily drawn on the existing literature:~\citet{du2020implicit} describe best practices for using EBM for generative modelling;~\citet{grathwohl2020your} explain how EBM can be used to train classifiers jointly for the discriminative and generative tasks. We have used the same package for training and inference, but there are some important differences between the two cases that are worth highlighting here.
-
-\subsubsection{Training: Joint Energy Models}
-
-To train our Joint Energy Models we broadly follow the approach outlined in~\citet{grathwohl2020your}. These models are trained to optimize a hybrid objective that involves a standard classification loss component $L_{\text{clf}}(\theta)=-\log p_{\theta}(\mathbf{y}|\mathbf{x})$ (e.g. cross-entropy loss) as well as a generative loss component $L_{\text{gen}}(\theta)=-\log p_{\theta}(\mathbf{x})$.
-
-To draw samples from $p_{\theta}(\mathbf{x})$, we rely exclusively on the conditional sampling approach described in~\citet{grathwohl2020your} for both training and inference: we first draw $\mathbf{y}\sim p(\mathbf{y})$ and then sample $\mathbf{x} \sim p_{\theta}(\mathbf{x}|\mathbf{y})$~\citep{grathwohl2020your} via Equation~\ref{eq:sgld} with energy $\mathcal{E}(\mathbf{x}|\mathbf{y})=\mu_{\theta}(\mathbf{x})[\mathbf{y}]$ where $\mu_{\theta}: \mathcal{X} \mapsto \mathbb{R}^K$ returns the linear predictions (logits) of our classifier $M_{\theta}$. While our package also supports unconditional sampling, we found conditional sampling to work well. It is also well aligned with CE, since in this context we are interested in conditioning on the target class.
-
-As mentioned in the body of the paper, we rely on a biased sampler involving separately specified values for the step size $\epsilon$ and the standard deviation $\sigma$ of the stochastic term involving $\mathbf{r}$. Formally, our biased sampler performs updates as follows:
-
-\begin{equation}\label{eq:biased-sgld}
- \begin{aligned}
- \hat{\mathbf{x}}_{j+1} &\leftarrow \hat{\mathbf{x}}_j - \frac{\epsilon}{2} \mathcal{E}(\hat{\mathbf{x}}_j|\mathbf{y}^+) + \sigma \mathbf{r}_j, && j=1,...,J
- \end{aligned}
-\end{equation}
-
-Consistent with~\citet{grathwohl2020your}, we have specified $\epsilon=2$ and $\sigma=0.01$ as the default values for all of our experiments. The number of total SGLD steps $J$ varies by dataset (Table~\ref{tab:ebmparams}). Following best practices, we initialize $\mathbf{x}_0$ randomly in 5\% of all cases and sample from a buffer in all other cases. The buffer itself is randomly initialised and gradually grows to a maximum of 10,000 samples during training as $\hat{\mathbf{x}}_{J}$ is stored in each epoch~\citep{du2020implicit,grathwohl2020your}.
-
-It is important to realise that sampling is done during each training epoch, which makes training Joint Energy Models significantly harder than conventional neural classifiers. In each epoch the generated (batch of) sample(s) $\hat{\mathbf{x}}_{J}$ is used as part of the generative loss component, which compares its energy to that of observed samples $\mathbf{x}$: $L_{\text{gen}}(\theta)=\mu_{\theta}(\mathbf{x})[\mathbf{y}]-\mu_{\theta}(\hat{\mathbf{x}}_{J})[\mathbf{y}]$. Our full training objective can be summarized as follows,
-
-\begin{equation}\label{eq:jem-loss}
- \begin{aligned}
- L(\theta) &= L_{\text{clf}}(\theta) + L_{\text{gen}}(\theta) + \lambda L_{\text{reg}}(\theta)
- \end{aligned}
-\end{equation}
-
-where $L_{\text{reg}}(\theta)$ is a Ridge penalty (L2 norm) that regularises energy magnitudes for both observed and generated samples~\citep{du2020implicit}. We have used varying degrees of regularization depending on the dataset ($\lambda$ in Table~\ref{tab:ebmparams}).
-
-Contrary to existing work, we have not typically used the entire minibatch of training data for the generative loss component but found that using a subset of the minibatch was often sufficient in attaining decent generative performance (Table~\ref{tab:ebmparams}). This has helped to reduce the computational burden for our models, which should make it easier for others to reproduce our findings. Figures~\ref{fig:mnist-gen} and~\ref{fig:moons-gen} show generated samples for our \textit{MNIST} and \textit{Moons} data, to provide a sense of their generative property.
-
-\import{contents/}{table_ebm_params.tex}
-
-\begin{figure}
- \centering
- \includegraphics[width=0.75\textwidth]{../artifacts/results/images/mnist_generated_JEM Ensemble.png}
- \caption{Conditionally generated \textit{MNIST} images for our JEM Ensemble.}\label{fig:mnist-gen}
-\end{figure}
-
-\begin{figure}
- \centering
- \includegraphics[width=0.5\textwidth]{../artifacts/results/images/moons_generated_JEM.png}
- \caption{Conditionally generated samples (stars) for our \textit{Moons} data using a JEM.}\label{fig:moons-gen}
-\end{figure}
-\subsubsection{Inference: Quantifying Models' Generative Property}
-
-At inference time, we assume no prior knowledge about the model's generative property. This means that we do not tab into the existing buffer of generated samples for our Joint Energy Models, but instead generate conditional samples from scratch. While we have relied on the default values $\epsilon=2$ and $\sigma=0.01$ also during inference, the number of total SGLD steps was set to $J=500$ in all cases, so significantly higher than during training. For all of our synthetic datasets and models, we generated 50 conditional samples and then formed subsets containing the $n_{E}=25$ lowest-energy samples. While in practice it would be sufficient to do this once for each model and dataset, we have chosen to perform sampling separately for each individual counterfactual in our experiments to account for stochasticity. To help reduce the computational burden for our real-world datasets we have generated only 10 conditional samples each time and used all of them in our counterfactual search. Using more samples, as we originally did, had no substantial impact on our results.
-
-\subsection{Conformal Prediction}\label{app:cp}
-
-In this Appendix~\ref{app:cp} we provide some more background on CP and explain in some more detail how we have used recent advances in Conformal Training for our purposes.
-
-\subsubsection{Background on CP}
-
-Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates by repeatedly sifting through the data. It can be used to generate prediction intervals for regression models and prediction sets for classification models. Since the literature on CE and AR is typically concerned with classification problems, we focus on the latter. A particular variant of CP called Split Conformal Prediction (SCP) is well-suited for our purposes, because it imposes only minimal restrictions on model training.
-
-Specifically, SCP involves splitting the data $\mathcal{D}_n=\{(\mathbf{x}_i,\mathbf{y}_i)\}_{i=1,...,n}$ into a proper training set $\mathcal{D}_{\text{train}}$ and a calibration set $\mathcal{D}_{\text{cal}}$. The former is used to train the classifier in any conventional fashion. The latter is then used to compute so-called nonconformity scores: $\mathcal{S}=\{s(\mathbf{x}_i,\mathbf{y}_i)\}_{i \in \mathcal{D}_{\text{cal}}}$ where $s: (\mathcal{X},\mathcal{Y}) \mapsto \mathbb{R}$ is referred to as \textit{score function}. In the context of classification, a common choice for the score function is just $s_i=1-M_{\theta}(\mathbf{x}_i)[\mathbf{y}_i]$, that is one minus the softmax output corresponding to the observed label $\mathbf{y}_i$~\citep{angelopoulos2021gentle}.
-
-Finally, classification sets are formed as follows,
-
-\begin{equation}\label{eq:scp}
- \begin{aligned}
- C_{\theta}(\mathbf{x}_i;\alpha)=\{\mathbf{y}: s(\mathbf{x}_i,\mathbf{y}) \le \hat{q}\}
- \end{aligned}
-\end{equation}
-
-where $\hat{q}$ denotes the $(1-\alpha)$-quantile of $\mathcal{S}$ and $\alpha$ is a predetermined error rate. As the size of the calibration set increases, the probability that the classification set $C(\mathbf{x}_{\text{test}})$ for a newly arrived sample $\mathbf{x}_{\text{test}}$ does not cover the true test label $\mathbf{y}_{\text{test}}$ approaches $\alpha$~\citep{angelopoulos2021gentle}.
-
-Observe from Equation~\ref{eq:scp} that Conformal Prediction works on an instance-level basis, much like CE are local. The prediction set for an individual instance $\mathbf{x}_i$ depends only on the characteristics of that sample and the specified error rate. Intuitively, the set is more likely to include multiple labels for samples that are difficult to classify, so the set size is indicative of predictive uncertainty. To see why this effect is exacerbated by small choices for $\alpha$ consider the case of $\alpha=0$, which requires that the true label is covered by the prediction set with probability equal to 1.
-
-\subsubsection{Differentiability}
-
-The fact that conformal classifiers produce set-valued predictions introduces a challenge: it is not immediately obvious how to use such classifiers in the context of gradient-based counterfactual search. Put differently, it is not clear how to use prediction sets in Equation~\ref{eq:general}. Fortunately, \citet{stutz2022learning} have recently proposed a framework for Conformal Training that also hinges on differentiability. Specifically, they show how Stochastic Gradient Descent can be used to train classifiers not only for the discriminative task but also for additional objectives related to Conformal Prediction. One such objective is \textit{efficiency}: for a given target error rate $\alpha$, the efficiency of a conformal classifier improves as its average prediction set size decreases. To this end, the authors introduce a smooth set size penalty defined in Equation~\ref{eq:setsize} in the body of this paper. Formally, it is defined as $C_{\theta,\mathbf{y}}(\mathbf{x}_i;\alpha):=\sigma\left((s(\mathbf{x}_i,\mathbf{y})-\alpha) T^{-1}\right)$ for $\mathbf{y}\in\mathcal{Y}$, where $\sigma$ is the sigmoid function and $T$ is a hyper-parameter used for temperature scaling~\citep{stutz2022learning}.
-
-In addition to the smooth set size penalty,~\citet{stutz2022learning} also propose a configurable classification loss function, that can be used to enforce coverage. For \textit{MNIST} data, we found that using this function generally improved the visual quality of the generated counterfactuals, so we used it in our experiments involving real-world data. For the synthetic dataset, visual inspection of the counterfactuals showed that using the configurable loss function sometimes led to overshooting: counterfactuals would end up deep inside the target domain but far away from the observed samples. For this reason, we instead relied on standard cross-entropy loss for our synthetic datasets. As we have noted in the body of the paper, more experimental work is certainly needed in this context. Figure~\ref{fig:cp-diff} shows the prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a \textit{JEM} trained on our \textit{Linearly Separable} data.
-
-\begin{figure}
- \centering
- \includegraphics[width=1.0\textwidth]{../artifacts/results/images/poc_set_size.png}
- \caption{Prediction set size (left), smooth set size loss (centre) and configurable classification loss (right) for a JEM trained on our \textit{Linearly Separable} data.}\label{fig:cp-diff}
-\end{figure}
-
-\subsection{ECCCo}\label{app:eccco}
-
-In this section, we briefly discuss convergence conditions for CE and provide details concerning the actual implementation of our framework in \texttt{Julia}.
-\subsubsection{A Note on Convergence}
-
-Convergence is not typically discussed much in the context of CE, even though it has important implications on outcomes. One intuitive way to specify convergence is in terms of threshold probabilities: once the predicted probability $p(\mathbf{y}^+|\mathbf{x}^{\prime})$ exceeds some user-defined threshold $\gamma$ such that the counterfactual is valid, we could consider the search to have converged. In the binary case, for example, convergence could be defined as $p(\mathbf{y}^+|\mathbf{x}^{\prime})>0.5$ in this sense. Note, however, how this can be expected to yield counterfactuals in the proximity of the decision boundary, a region characterized by high aleatoric uncertainty. In other words, counterfactuals generated in this way would generally not be plausible. To avoid this from happening, we specify convergence in terms of gradients approaching zero for all our experiments and all of our generators. This is allows us to get a cleaner read on how the different counterfactual search objectives affect counterfactual outcomes.
-
-\subsubsection{\texttt{ECCCo.jl}}
-
-The core part of our code base is integrated into a larger ecosystem of \texttt{Julia} packages that we are actively developing and maintaining. To avoid compromising the double-blind review process, we only provide a link to an anonymized repository at this stage: \url{https://anonymous.4open.science/r/ECCCo-1252/README.md}.
-
-\subsection{Experimental Setup}\label{app:setup}
-
-Table~\ref{tab:params} provides an overview of all parameters related to our experiments. The \textit{GMSC} data were randomly undersampled for balancing purposes and all features were standardized. \textit{MNIST} data was also randomly undersampled for reasons outlined below. Pixel values were preprocessed to fall in the range of $[-1,1]$ and a small Gaussian noise component ($\sigma=0.03$) was added to training samples following common practice in the EBM literature. All of our models were trained through mini-batch training using the Adam optimiser (\citet{kingma2017adam}). Table~\ref{tab:perf} shows standard evaluation metrics measuring the predictive performance of our different models grouped by dataset. These measures were computed on test data.
-
-Table~\ref{tab:genparams} summarises our hyperparameter choices for the counterfactual generators where $\eta$ denotes the learning rate used for Stochastic Gradient Descent (SGD) and $\lambda_1$, $\lambda_2$, $\lambda_3$ represent the chosen penalty strengths (Equations~\ref{eq:general} and~\ref{eq:eccco}). Here $\lambda_1$ also refers to the chosen penalty for the distance from factual values that applies to both \textit{Wachter} and \textit{REVISE}, but not \textit{Schut} which is penalty-free. \textit{Schut} is also the only generator that uses JSMA instead of SGD for optimization.
-
-\import{contents/}{table_params.tex}
-
-\import{contents/}{table_perf.tex}
-
-\import{contents/}{table_gen_params.tex}
-
-\subsubsection{Compute}
-
-To enable others to easily replicate our experiments, we have chosen to work with small neural network architectures and randomly undersampled the \textit{MNIST} dataset (maintaining class balance). All of our experiments could then be run locally on a personal machine. The longest runtimes we experienced for model training and counterfactual benchmarking were on the order of 8-12 hours (\textit{MNIST} data). For the synthetic data, all experiments could be completed in less than an hour.
-
-We have summarised our system information below:
-
-\textbf{Software}:
-
-\begin{itemize}
- \item System Version: macOS 13.3.1
- \item Kernel Version: Darwin 22.4.0
-\end{itemize}
-
-\textbf{Hardware}:
-
-\begin{itemize}
- \item Model Name: MacBook Pro
- \item Model Identifier: MacBookPro16,1
- \item Processor Name: 8-Core Intel Core i9
- \item Processor Speed: 2.3 GHz
- \item Number of Processors: 1
- \item Total Number of Cores: 8
- \item L2 Cache (per Core): 256 KB
- \item L3 Cache: 16 MB
- \item Hyper-Threading Technology: Enabled
- \item Memory: 32 GB
-\end{itemize}
-
-
-\subsection{Results}\label{app:results}
-
-Figure~\ref{fig:mnist-eccco} shows examples of counterfactuals for \textit{MNIST} data where the underlying model is our \textit{JEM Ensemble}. Original images are shown on the diagonal and the corresponding counterfactuals are plotted across rows.
-
-\begin{figure}
- \centering
- \includegraphics[width=0.9\textwidth]{../artifacts/results/images/mnist_eccco_all_digits.png}
- \caption{Counterfactuals for \textit{MNIST} data and our \textit{JEM Ensemble}. Original images are shown on the diagonal with the corresponding counterfactuals plotted across rows.}\label{fig:mnist-eccco}
-\end{figure}
-
-Table~\ref{tab:results-full} reports all of the evaluation metrics we have computed. Table~\ref{tab:results-full-valid} reports the same metrics for the subset of valid counterfactuals. The `Unfaithfulness' and `Implausibility' metrics have been discussed extensively in the body of the paper. The `Cost' metric relates to the distance between the factual and the counterfactual. The `Redundancy' metric measures sparsity in is defined as the percentage of features that remain unperturbed (higher is better). The `Uncertainty' metric is just the average value of the smooth set size penalty (Equation~\ref{eq:setsize}). Finally, `Validity' is the percentage of valid counterfactuals.
-
-\import{contents/}{table_all.tex}
-
-\import{contents/}{table_all_valid.tex}
-
-
-\end{document}
\ No newline at end of file
diff --git a/paper/tables.Rmd b/paper/tables.Rmd
index 487ae3dd07cae35bc1ba4238db3361e60b54d1eb..21cde35abedea2075f31443e128f1d4a9c9e0a7c 100644
--- a/paper/tables.Rmd
+++ b/paper/tables.Rmd
@@ -44,8 +44,9 @@ dt_valid <- dt[non_valid==FALSE]
```
```{r}
+generators <- unique(dt$generator)[sapply(unique(dt$generator), function(x) {!grepl("L1",x)})]
tab <- dt[
- ,
+ generator %in% generators,
.(
value=sprintf("%1.2f ± %1.2f", mean(value), sd(value)),
val = mean(value),
@@ -100,8 +101,9 @@ tab[,one_std_wachter:=NULL]
```
```{r}
+generators <- unique(dt$generator)[sapply(unique(dt$generator), function(x) {!grepl("L1",x)})]
tab_valid <- dt_valid[
- ,
+ generator %in% generators,
.(
value=sprintf("%1.2f ± %1.2f", mean(value), sd(value)),
val = mean(value),
@@ -161,84 +163,33 @@ tab_valid[,one_std_wachter:=NULL]
# Choices:
measures <- c(
"unfaithfulness",
- "implausibility"
+ "implausibility",
+ "set_size_penalty"
)
measure_names <- c(
"Unfaithfulness ↓",
- "Implausibility ↓"
+ "Implausibility ↓",
+ "Uncertainty ↓"
)
+chosen_source <- "tabular"
# Order:
chosen_data <- c(
- "Linearly Separable",
- "GMSC",
- "MNIST",
+ "California Housing",
+ "GMSC"
)
chosen_model <- c(
"MLP",
- "JEM",
- "LeNet-5"
+ "JEM Ensemble"
)
tab_i <- tab
# Logic:
tab_i <- tab_i[variable %in% measures]
+tab_i[,variable:=factor(variable, levels=measures)]
tab_i <- tab_i[dataname %in% chosen_data]
tab_i <- tab_i[model %in% chosen_model]
-tab_i[,dataname:=factor(dataname,levels=chosen_data)]
-tab_i <- dcast(tab_i, model + generator ~ dataname + variable)
-col_names <- c(
- "Model",
- "Generator",
- rep(measure_names,length(chosen_data))
-)
-caption <- "Results for datasets from different domains: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-main} \\newline"
-file_name <- "paper/contents/table-main.tex"
-sub_header <- rep(length(measures), length(chosen_data))
-names(sub_header) <- chosen_data
-header <- c(
- " " = 2, sub_header
-)
-line_sep <- c(rep("",length(measures)-1),"\\addlinespace")
-algin_cols <- c(rep('l',2),rep('c',ncol(tab_i)-2))
-kbl(
- tab_i, caption = caption,
- align = algin_cols, col.names=col_names, booktabs = T, escape=F,
- format="latex", linesep = line_sep
-) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = T) %>%
- add_header_above(header) %>%
- collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
- save_kable(file_name)
-```
-
-```{r}
-# Choices:
-measures <- c(
- "distance_from_energy_l2",
- "distance_from_targets_l2"
-)
-measure_names <- c(
- "Unfaithfulness ↓",
- "Implausibility ↓"
-)
-chosen_source <- "tabular"
-# Order:
-chosen_data <- c(
- "GMSC",
- "California Housing",
- "German Credit"
-)
-chosen_model <- "MLP"
-
-tab_i <- tab
-
-# Logic:
-tab_i <- tab_i[variable %in% measures]
-tab_i <- tab_i[source == chosen_source]
-tab_i <- tab_i[dataname %in% chosen_data]
-tab_i <- tab_i[model == chosen_model]
+tab_i[,model:=factor(model,levels=chosen_model)]
tab_i[,dataname:=factor(dataname,levels=chosen_data)]
tab_i <- dcast(tab_i, model + generator ~ dataname + variable)
col_names <- c(
@@ -247,7 +198,7 @@ col_names <- c(
rep(measure_names,length(chosen_data))
)
caption <- sprintf(
- "Results for %s datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-%s} \\newline",
+ "Results for %s datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\\textit{Wachter}). \\label{tab:results-%s} \\newline",
chosen_source,
chosen_source
)
@@ -265,10 +216,12 @@ algin_cols <- c(rep('l',2),rep('c',ncol(tab_i)-2))
kbl(
tab_i, caption = caption,
align = algin_cols, col.names=col_names, booktabs = T, escape=F,
- format="latex", linesep = line_sep
+ format="latex", linesep = line_sep, table.env="table*"
) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = T) %>%
+ kable_styling(
+ latex_options = c("scale_down")
+ ) %>%
+ kable_paper(full_width = F) %>%
add_header_above(header) %>%
collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
save_kable(file_name)
@@ -276,27 +229,32 @@ kbl(
```{r}
# Choices:
-# Choices:
measures <- c(
- "distance_from_energy_l2",
- "distance_from_targets_l2"
+ "unfaithfulness",
+ "implausibility"
)
measure_names <- c(
"Unfaithfulness ↓",
"Implausibility ↓"
)
-chosen_source <- "synthetic"
+chosen_source <- "vision"
+# Order:
chosen_data <- c(
- "Linearly Separable",
- "Moons",
- "Circles"
+ "MNIST"
+)
+chosen_model <- c(
+ "MLP",
+ "LeNet-5"
)
+
tab_i <- tab
# Logic:
tab_i <- tab_i[variable %in% measures]
-tab_i <- tab_i[source == chosen_source]
+tab_i[,variable:=factor(variable, levels=measures)]
tab_i <- tab_i[dataname %in% chosen_data]
+tab_i <- tab_i[model %in% chosen_model]
+tab_i[,model:=factor(model,levels=chosen_model)]
tab_i[,dataname:=factor(dataname,levels=chosen_data)]
tab_i <- dcast(tab_i, model + generator ~ dataname + variable)
col_names <- c(
@@ -305,7 +263,7 @@ col_names <- c(
rep(measure_names,length(chosen_data))
)
caption <- sprintf(
- "Results for %s datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-%s} \\newline",
+ "Results for %s dataset. Formatting details are the same as in Table~\\ref{tab:results-tabular}. \\label{tab:results-%s} \\newline",
chosen_source,
chosen_source
)
@@ -325,133 +283,244 @@ kbl(
align = algin_cols, col.names=col_names, booktabs = T, escape=F,
format="latex", linesep = line_sep
) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = T) %>%
+ kable_styling(
+ latex_options = c("scale_down")
+ ) %>%
+ kable_paper(full_width = F) %>%
add_header_above(header) %>%
collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
save_kable(file_name)
```
+## Full tables
+
```{r}
-# Choices:
+tab <- dt[
+ ,
+ .(
+ value=sprintf("%1.2f ± %1.2f", mean(value), sd(value)),
+ val = mean(value),
+ std = sd(value)
+ ),
+ .(dataname, generator, model, variable, source)
+]
+
+tab$top_val = F
+tab$one_std_wachter = F
+tab$two_std_wachter = F
+
+# Measures to be minimized:
+min_measures <- c(
+ "distance",
+ "implausibility",
+ "unfaithfulness",
+ "distance_from_energy",
+ "distance_from_energy_l2",
+ "distance_from_targets",
+ "distance_from_targets_l2",
+ "set_size_penalty"
+)
+tab[variable %in% min_measures,top_val:=val==min(val),.(model, dataname, variable)]
+tab[variable %in% min_measures,top_val:=ifelse(rep(all(top_val),length(top_val)),F,top_val),.(model, dataname, variable)]
+tab[variable %in% min_measures,two_std_wachter:=val+2*std<val[generator=="Wachter"],.(model, dataname, variable)]
+tab[variable %in% min_measures,one_std_wachter:=val+1*std<val[generator=="Wachter"],.(model, dataname, variable)]
+
+# Measures to be maximized:
+max_measures <- c(
+ "validity",
+ "redundancy"
+)
+tab[variable %in% max_measures,top_val:=val==max(val),.(model, dataname, variable)]
+tab[variable %in% max_measures,top_val:=ifelse(rep(all(top_val),length(top_val)),F,top_val),.(model, dataname, variable)]
+tab[variable %in% max_measures,two_std_wachter:=val-2*std>val[generator=="Wachter"],.(model, dataname, variable)]
+tab[variable %in% max_measures,one_std_wachter:=val-1*std>val[generator=="Wachter"],.(model, dataname, variable)]
+
+# Add conditional formatting:
+tab$value <- cell_spec(tab$value, "latex", bold=tab$top_val)
+tab[one_std_wachter==T,value:=paste0(value,"*")]
+tab[one_std_wachter==F,value:=paste0(value,"\\hphantom{*}")]
+tab[two_std_wachter==T,value:=paste0(value,"*")]
+tab[two_std_wachter==F,value:=paste0(value,"\\hphantom{*}")]
+
+# Remove redundant columns:
+tab[,val:=NULL]
+tab[,std:=NULL]
+tab[,top_val:=NULL]
+tab[,two_std_wachter:=NULL]
+tab[,one_std_wachter:=NULL]
+
+# Measures:
measures <- c(
- "distance_from_energy_l2",
- "distance_from_targets_l2"
+ "unfaithfulness",
+ "implausibility",
+ "set_size_penalty",
+ "distance",
+ "redundancy",
+ "validity"
)
measure_names <- c(
"Unfaithfulness ↓",
- "Implausibility ↓"
-)
-chosen_source <- "vision"
-# Order:
-chosen_data <- c(
- "MNIST",
- "Fashion MNIST",
+ "Implausibility ↓",
+ "Uncertainty ↓",
+ "Cost ↓",
+ "Redundancy ↑",
+ "Validity ↑"
)
-chosen_model <- "LeNet-5"
+tab <- tab[variable %in% measures]
+tab[,variable:=factor(variable, levels=measures)]
+```
-tab_i <- tab
+```{r}
+tab_valid <- dt_valid[
+ ,
+ .(
+ value=sprintf("%1.2f ± %1.2f", mean(value), sd(value)),
+ val = mean(value),
+ std = sd(value)
+ ),
+ .(dataname, generator, model, variable, source)
+]
-# Logic:
-tab_i <- tab_i[variable %in% measures]
-tab_i <- tab_i[source == chosen_source]
-tab_i <- tab_i[dataname %in% chosen_data]
-tab_i <- tab_i[model == chosen_model]
-tab_i[,dataname:=factor(dataname,levels=chosen_data)]
-tab_i <- dcast(tab_i, model + generator ~ dataname + variable)
-col_names <- c(
- "Model",
- "Generator",
- rep(measure_names,length(chosen_data))
+tab_valid$top_val = F
+tab_valid$one_std_wachter = F
+tab_valid$two_std_wachter = F
+
+# Measures to be minimized:
+min_measures <- c(
+ "distance",
+ "implausibility",
+ "unfaithfulness",
+ "distance_from_energy",
+ "distance_from_energy_l2",
+ "distance_from_targets",
+ "distance_from_targets_l2",
+ "set_size_penalty"
)
-caption <- sprintf(
- "Results for %s datasets: sample averages +/- one standard deviation across counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-%s} \\newline",
- chosen_source,
- chosen_source
+tab_valid[variable %in% min_measures,top_val:=val==min(val),.(model, dataname, variable)]
+tab_valid[variable %in% min_measures,top_val:=ifelse(rep(all(top_val),length(top_val)),F,top_val),.(model, dataname, variable)]
+tab_valid[variable %in% min_measures,two_std_wachter:=val+2*std<val[generator=="Wachter"],.(model, dataname, variable)]
+tab_valid[variable %in% min_measures,one_std_wachter:=val+1*std<val[generator=="Wachter"],.(model, dataname, variable)]
+
+# Measures to be maximized:
+max_measures <- c(
+ "validity",
+ "redundancy"
)
-file_name <- sprintf(
- "paper/contents/table-%s.tex",
- chosen_source
+tab_valid[variable %in% max_measures,top_val:=val==max(val),.(model, dataname, variable)]
+tab_valid[variable %in% max_measures,top_val:=ifelse(rep(all(top_val),length(top_val)),F,top_val),.(model, dataname, variable)]
+tab_valid[variable %in% max_measures,two_std_wachter:=val-2*std>val[generator=="Wachter"],.(model, dataname, variable)]
+tab_valid[variable %in% max_measures,one_std_wachter:=val-1*std>val[generator=="Wachter"],.(model, dataname, variable)]
+
+# Add conditional formatting:
+tab_valid$value <- cell_spec(tab_valid$value, "latex", bold=tab_valid$top_val)
+tab_valid[one_std_wachter==T,value:=paste0(value,"*")]
+tab_valid[one_std_wachter==F,value:=paste0(value,"\\hphantom{*}")]
+tab_valid[two_std_wachter==T,value:=paste0(value,"*")]
+tab_valid[two_std_wachter==F,value:=paste0(value,"\\hphantom{*}")]
+
+# Remove redundant columns:
+tab_valid[,val:=NULL]
+tab_valid[,std:=NULL]
+tab_valid[,top_val:=NULL]
+tab_valid[,two_std_wachter:=NULL]
+tab_valid[,one_std_wachter:=NULL]
+
+# Measures:
+measures <- c(
+ "unfaithfulness",
+ "implausibility",
+ "set_size_penalty",
+ "distance",
+ "redundancy",
+ "validity"
)
-sub_header <- rep(length(measures), length(chosen_data))
-names(sub_header) <- chosen_data
-header <- c(
- " " = 2, sub_header
+measure_names <- c(
+ "Unfaithfulness ↓",
+ "Implausibility ↓",
+ "Uncertainty ↓",
+ "Cost ↓",
+ "Redundancy ↑",
+ "Validity ↑"
)
-line_sep <- c(rep("",length(measures)-1),"\\addlinespace")
-algin_cols <- c(rep('l',2),rep('c',ncol(tab_i)-2))
-kbl(
- tab_i, caption = caption,
- align = algin_cols, col.names=col_names, booktabs = T, escape=F,
- format="latex", linesep = line_sep
-) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = T) %>%
- add_header_above(header) %>%
- collapse_rows(columns = 1:2, latex_hline = "major", valign = "middle") %>%
- save_kable(file_name)
+tab_valid <- tab_valid[variable %in% measures]
+tab_valid[,variable:=factor(variable, levels=measures)]
```
-## Full table
+### Tables (all)
```{r}
-tab_full <- dcast(tab, dataname + model + generator ~ variable)
-col_names <- c(
- "Model",
- "Data",
- "Generator",
- "Cost ↓",
- "Unfaithfulness ↓",
- "Implausibility ↓",
- "Redundancy ↑",
- "Uncertainty ↓",
- "Validity ↑"
-)
-algin_cols <- c(rep('l',3),rep('c',ncol(tab_full)-3))
-kbl(
- tab_full, caption = "All results for all datasets: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-full} \\newline",
- align = "c", col.names=col_names, booktabs = T, escape=F,
- format="latex"
-) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
- collapse_rows(columns = 1:3, latex_hline = "custom", valign = "top", custom_latex_hline = 1:2) %>%
- save_kable("paper/contents/table_all.tex")
+for (name in unique(tab$dataname)) {
+ data_indicator <- gsub(" ", "-", tolower(unique(name)))
+ # Choices:
+ tab_full <- dcast(tab[dataname==name], model + generator ~ variable)
+ col_names <- c(
+ "Model",
+ "Generator",
+ measure_names
+ )
+ algin_cols <- c(rep('l',3),rep('c',ncol(tab_full)-3))
+ file_name <- sprintf(
+ "paper/contents/table-%s.tex",
+ data_indicator
+ )
+ cap <- sprintf(
+ "All results for %s dataset: sample averages +/- one standard deviation over all counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\\textit{Wachter}). \\label{tab:results-%s} \\newline",
+ name,
+ data_indicator
+ )
+ kbl(
+ tab_full, caption = cap,
+ align = "c", col.names=col_names, booktabs = T, escape=F,
+ format="latex"
+ ) %>%
+ kable_styling(latex_options = c("scale_down")) %>%
+ kable_paper(full_width = F) %>%
+ collapse_rows(columns = 1:2, latex_hline = "custom", valign = "top") %>%
+ save_kable(file_name)
+}
+
```
## Full table (valid only)
```{r}
-tab_full <- dcast(tab_valid, dataname + model + generator ~ variable)
-col_names <- c(
- "Model",
- "Data",
- "Generator",
- "Cost ↓",
- "Unfaithfulness ↓",
- "Implausibility ↓",
- "Redundancy ↑",
- "Uncertainty ↓",
- "Validity ↑"
-)
-algin_cols <- c(rep('l',3),rep('c',ncol(tab_full)-3))
-kbl(
- tab_full, caption = "All results for all datasets: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (Wachter). \\label{tab:results-full-valid} \\newline",
- align = "c", col.names=col_names, booktabs = T, escape=F,
- format="latex"
-) %>%
- kable_styling(latex_options = c("scale_down")) %>%
- kable_paper(full_width = F) %>%
- collapse_rows(columns = 1:3, latex_hline = "custom", valign = "top", custom_latex_hline = 1:2) %>%
- save_kable("paper/contents/table_all_valid.tex")
+for (name in unique(tab_valid$dataname)) {
+ data_indicator <- gsub(" ", "-", tolower(unique(name)))
+ # Choices:
+ tab_full <- dcast(tab_valid[dataname==name], model + generator ~ variable)
+ col_names <- c(
+ "Model",
+ "Generator",
+ measure_names
+ )
+ algin_cols <- c(rep('l',3),rep('c',ncol(tab_full)-3))
+ file_name <- sprintf(
+ "paper/contents/table-%s-valid.tex",
+ data_indicator
+ )
+ cap <- sprintf(
+ "All results for %s dataset: sample averages +/- one standard deviation over all valid counterfactuals. Best outcomes are highlighted in bold. Asterisks indicate that the given value is more than one (*) or two (**) standard deviations away from the baseline (\\textit{Wachter}). \\label{tab:results-%s-valid} \\newline",
+ name,
+ data_indicator
+ )
+ kbl(
+ tab_full, caption = cap,
+ align = "c", col.names=col_names, booktabs = T, escape=F,
+ format="latex"
+ ) %>%
+ kable_styling(latex_options = c("scale_down")) %>%
+ kable_paper(full_width = F) %>%
+ collapse_rows(columns = 1:2, latex_hline = "custom", valign = "top") %>%
+ save_kable(file_name)
+}
```
## EBM
```{r}
-files <- list.files("artifacts/params/")
-dt <- lapply(files[grepl(".csv", files)], function(x) {
- fread(file.path("artifacts/params/", x))
+files <- list.files(paste0(res_path,"params"))
+dt <- lapply(files[sapply(files, function(x) grepl("model_params.csv",x))], function(x) {
+ fread(file.path(paste0(res_path,"params"), x))
})
dt <- Reduce(function(x,y) {rbind(x,y, fill=TRUE)}, dt)
setcolorder(
@@ -463,7 +532,12 @@ setcolorder(
"jem_sampling_steps", "sgld_batch_size", "lambda"
)
)
-dt[,dataname:=factor(dataname, levels=c("Linearly Separable", "Moons", "Circles", "MNIST", "GMSC"))]
+dataset_order = c(
+ "Linearly Separable", "Moons", "Circles",
+ "California Housing", "GMSC", "German Credit",
+ "MNIST", "Fashion MNIST"
+)
+dt[,dataname:=factor(dataname, levels=dataset_order)]
dt <- dt[order(dataname)]
dt_ebm <- dt[,.(dataname, jem_sampling_steps, sgld_batch_size, lambda)]
col_names <- c(
@@ -473,7 +547,7 @@ col_names <- c(
kbl(
dt_ebm, caption = "EBM hyperparemeter choices for our experiments. \\label{tab:ebmparams} \\newline",
align = "r", col.names=col_names, booktabs = T, escape=F,
- format="latex"
+ format="latex", linesep = ""
) %>%
kable_styling(font_size = 8) %>%
kable_paper(full_width = F) %>%
@@ -493,7 +567,7 @@ header <- c(" " = 2, "Network Architecture" = 4, "Training" = 2)
kbl(
dt_exp, caption = "Paremeter choices for our experiments. \\label{tab:params} \\newline",
align = "r", col.names=col_names, booktabs = T, escape=F,
- format="latex"
+ format="latex", linesep = ""
) %>%
kable_styling(latex_options = c("scale_down")) %>%
kable_paper(full_width = F) %>%
@@ -502,13 +576,12 @@ kbl(
```
```{r}
-files <- list.files("artifacts/params/generator")
-dt <- lapply(files, function(x) {
- fread(file.path("artifacts/params/generator", x))
+dt <- lapply(files[sapply(files, function(x) grepl("generator_params.csv",x))], function(x) {
+ fread(file.path(paste0(res_path,"params"), x))
})
dt <- Reduce(function(x,y) {rbind(x,y, fill=TRUE)}, dt)
-dt <- dt[,.(dataname,eta,λ1,λ3,λ3)]
-dt[,dataname:=factor(dataname, levels=c("Linearly Separable", "Moons", "Circles", "MNIST", "GMSC"))]
+dt <- dt[,.(dataname,eta,lambda_1,lambda_2,lambda_3)]
+dt[,dataname:=factor(dataname, levels=dataset_order)]
dt <- dt[order(dataname)]
col_names <- c(
"Dataset",
@@ -517,7 +590,7 @@ col_names <- c(
kbl(
dt, caption = "Generator hyperparameters. \\label{tab:genparams} \\newline",
align = "r", col.names=col_names, booktabs = T, escape=F,
- format="latex"
+ format="latex", linesep = ""
) %>%
kable_styling(font_size = 8) %>%
kable_paper(full_width = F) %>%
@@ -525,12 +598,11 @@ kbl(
```
```{r}
-files <- list.files("artifacts/results/")
dt <- lapply(files[grepl("_model_performance.csv", files)], function(x) {
- fread(file.path("artifacts/results/", x))
+ fread(file.path(paste0(res_path,"params"), x))
})
dt <- Reduce(function(x,y) {rbind(x,y, fill=TRUE)}, dt)
-dt[,dataname:=factor(dataname, levels=c("Linearly Separable", "Moons", "Circles", "MNIST", "GMSC"))]
+dt[,dataname:=factor(dataname, levels=dataset_order)]
dt <- dt[order(dataname,mod_name)]
setcolorder(
dt,
@@ -542,8 +614,8 @@ setcolorder(
col_names <- c("Dataset", "Model", "Accuracy", "Precision", "F1-Score")
kbl(
dt, caption = "Various standard performance metrics for our different models grouped by dataset. \\label{tab:perf} \\newline",
- align = "r", col.names=col_names, booktabs = T, escape=F,
- format="latex", digits=2
+ align = "r", col.names=col_names, booktabs = T, escape=F,
+ format="latex", digits=2, linesep = ""
) %>%
kable_styling(font_size = 8) %>%
kable_paper(full_width = F) %>%
diff --git a/results_mpi/linearly_separable_bmk.jls b/results_mpi/linearly_separable_bmk.jls
new file mode 100644
index 0000000000000000000000000000000000000000..281021256679d855e18b95c9afb00fc0fa7cb2de
Binary files /dev/null and b/results_mpi/linearly_separable_bmk.jls differ
diff --git a/results_mpi/linearly_separable_models.jls b/results_mpi/linearly_separable_models.jls
new file mode 100644
index 0000000000000000000000000000000000000000..cc0f65d7f227b0f52e2b4705972ddd8e8f94a225
Binary files /dev/null and b/results_mpi/linearly_separable_models.jls differ
diff --git a/results_mpi/linearly_separable_outcome.jls b/results_mpi/linearly_separable_outcome.jls
new file mode 100644
index 0000000000000000000000000000000000000000..272c71a8b84a20687a436de07411d99e6a2c2f23
Binary files /dev/null and b/results_mpi/linearly_separable_outcome.jls differ
diff --git a/results_mpi/params/linearly_separable_generator_params.csv b/results_mpi/params/linearly_separable_generator_params.csv
new file mode 100644
index 0000000000000000000000000000000000000000..75b36812fed250ba127532675b0810edfba941f0
--- /dev/null
+++ b/results_mpi/params/linearly_separable_generator_params.csv
@@ -0,0 +1,2 @@
+dataname,eta,lambda_1,lambda_1_Δ,lambda_2,lambda_2_Δ,lambda_3,lambda_3_Δ,n_individuals,opt
+Linearly Separable,0.01,0.25,0.25,0.75,0.75,0.75,0.75,25,Descent
diff --git a/results_mpi/params/linearly_separable_model_params.csv b/results_mpi/params/linearly_separable_model_params.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4b02f8a648255130f92718fe46bb9194fa15399b
--- /dev/null
+++ b/results_mpi/params/linearly_separable_model_params.csv
@@ -0,0 +1,2 @@
+activation,batch_size,dataname,epochs,jem_sampling_steps,lambda,n_ens,n_hidden,n_layers,n_obs,sgld_batch_size
+relu,100,Linearly Separable,100,50,0.1,5,32,3,1000,50
diff --git a/results_mpi/params/linearly_separable_model_performance.csv b/results_mpi/params/linearly_separable_model_performance.csv
new file mode 100644
index 0000000000000000000000000000000000000000..08598e7bcea673ebc02bcdf08eb17b906b3f2ec9
--- /dev/null
+++ b/results_mpi/params/linearly_separable_model_performance.csv
@@ -0,0 +1,5 @@
+acc,precision,f1score,mod_name,dataname
+0.992,0.992,0.992,JEM Ensemble,Linearly Separable
+0.992,0.992,0.992,MLP,Linearly Separable
+0.992,0.992,0.992,MLP Ensemble,Linearly Separable
+0.988,0.98828125,0.9879982717511322,JEM,Linearly Separable
diff --git a/results_mpi/params/linearly_separable_model_performance.jls b/results_mpi/params/linearly_separable_model_performance.jls
new file mode 100644
index 0000000000000000000000000000000000000000..f0bf64f479bfdd67421291d1eeb6d8d56fe429a3
Binary files /dev/null and b/results_mpi/params/linearly_separable_model_performance.jls differ
diff --git a/src/penalties.jl b/src/penalties.jl
index 6e7dad339917507c47467eb66a3983cd1f20a14f..2e61109712a25df6dcf8e6af66ccbbd42d437518 100644
--- a/src/penalties.jl
+++ b/src/penalties.jl
@@ -48,6 +48,7 @@ function energy_delta(
nmin::Int=25,
return_conditionals=false,
reg_strength=0.1,
+ decay::Real=0.1,
kwargs...
)
diff --git a/www/mnist_all_jem_eccco.png b/www/mnist_all_jem_eccco.png
new file mode 100644
index 0000000000000000000000000000000000000000..7cb52db3fa3384934273984153eb8d8090eb869a
Binary files /dev/null and b/www/mnist_all_jem_eccco.png differ
diff --git a/www/mnist_all_jem_ens_eccco.png b/www/mnist_all_jem_ens_eccco.png
new file mode 100644
index 0000000000000000000000000000000000000000..03b5d45786a304784f97d8f8a80b4fef8db46f23
Binary files /dev/null and b/www/mnist_all_jem_ens_eccco.png differ
diff --git a/www/mnist_all_jem_ens_revise.png b/www/mnist_all_jem_ens_revise.png
new file mode 100644
index 0000000000000000000000000000000000000000..555a1226fafd0be6853d8de2f95427ba66fd1a8a
Binary files /dev/null and b/www/mnist_all_jem_ens_revise.png differ
diff --git a/www/mnist_all_jem_revise.png b/www/mnist_all_jem_revise.png
new file mode 100644
index 0000000000000000000000000000000000000000..1aa1df5411ebd7869971628b57f888e24b5189f9
Binary files /dev/null and b/www/mnist_all_jem_revise.png differ
diff --git a/www/mnist_all_lenet_eccco.png b/www/mnist_all_lenet_eccco.png
new file mode 100644
index 0000000000000000000000000000000000000000..18daf27df2c8254f25287b1bce58a077f2062738
Binary files /dev/null and b/www/mnist_all_lenet_eccco.png differ
diff --git a/www/mnist_all_lenet_revise.png b/www/mnist_all_lenet_revise.png
new file mode 100644
index 0000000000000000000000000000000000000000..85c19af5e69d7b53e8c54f1c9591dbb2f04284ae
Binary files /dev/null and b/www/mnist_all_lenet_revise.png differ
diff --git a/www/mnist_all_mlp.png b/www/mnist_all_mlp.png
new file mode 100644
index 0000000000000000000000000000000000000000..9c27ab6c38a45e8f00339e2e142075c0122b5627
Binary files /dev/null and b/www/mnist_all_mlp.png differ
diff --git a/www/mnist_all_mlp_eccco.png b/www/mnist_all_mlp_eccco.png
new file mode 100644
index 0000000000000000000000000000000000000000..9c27ab6c38a45e8f00339e2e142075c0122b5627
Binary files /dev/null and b/www/mnist_all_mlp_eccco.png differ
diff --git a/www/mnist_all_mlp_ens_eccco.png b/www/mnist_all_mlp_ens_eccco.png
new file mode 100644
index 0000000000000000000000000000000000000000..6b4c978fb82c52ff514dc15de1250601f31dbab4
Binary files /dev/null and b/www/mnist_all_mlp_ens_eccco.png differ
diff --git a/www/mnist_all_mlp_ens_revise.png b/www/mnist_all_mlp_ens_revise.png
new file mode 100644
index 0000000000000000000000000000000000000000..f98db2820a69e360d6c0ad1e39e7ecff2176c040
Binary files /dev/null and b/www/mnist_all_mlp_ens_revise.png differ
diff --git a/www/mnist_all_mlp_revise.png b/www/mnist_all_mlp_revise.png
new file mode 100644
index 0000000000000000000000000000000000000000..200d3d78c84a7c87b5056693b16a68124f0ed046
Binary files /dev/null and b/www/mnist_all_mlp_revise.png differ
diff --git a/www/mnist_benchmark.png b/www/mnist_benchmark.png
new file mode 100644
index 0000000000000000000000000000000000000000..15321de317af13d6dc2483eb3248711cff01b894
Binary files /dev/null and b/www/mnist_benchmark.png differ
diff --git a/www/vae_rec.png b/www/vae_rec.png
new file mode 100644
index 0000000000000000000000000000000000000000..dc11d1aa1db7bdebbe93b12bf11448901b9b8d0d
Binary files /dev/null and b/www/vae_rec.png differ