diff --git a/Manifest.toml b/Manifest.toml
index d2e997cf1ad3de8f5cfa790e14dd6ede8e6a289b..50b8fe3a241474c1800a590a1dcbc66f8123e255 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -1,6 +1,6 @@
# This file is machine-generated - editing it directly is not advised
-julia_version = "1.8.5"
+julia_version = "1.6.5"
manifest_format = "2.0"
project_hash = "d4865488d9d741cc2d833e66a476d21fceebe412"
@@ -10,6 +10,11 @@ git-tree-sha1 = "16b6dbc4cf7caee4e1e75c49485ec67b667098a0"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "1.3.1"
+[[deps.AbstractTrees]]
+git-tree-sha1 = "faa260e4cb5aba097a73fab382dd4b5819d8ec8c"
+uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+version = "0.4.4"
+
[[deps.Accessors]]
deps = ["Compat", "CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "LinearAlgebra", "MacroTools", "Requires", "StaticArrays", "Test"]
git-tree-sha1 = "beabc31fa319f9de4d16372bff31b4801e43d32c"
@@ -29,7 +34,6 @@ version = "2.3.0"
[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
-version = "1.1.1"
[[deps.ArnoldiMethod]]
deps = ["LinearAlgebra", "Random", "StaticArrays"]
@@ -44,10 +48,10 @@ uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97"
version = "0.5.4"
[[deps.Arpack_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "OpenBLAS_jll", "Pkg"]
-git-tree-sha1 = "5ba6c757e8feccf03a1554dfaf3e26b3cfc7fd5e"
+deps = ["Libdl", "OpenBLAS_jll", "Pkg"]
+git-tree-sha1 = "e214a9b9bd1b4e1b4f15b22c0994862b66af7ff7"
uuid = "68821587-b530-5797-8361-c406ea357684"
-version = "3.5.1+1"
+version = "3.5.0+3"
[[deps.ArrayInterface]]
deps = ["Adapt", "LinearAlgebra", "Requires", "SnoopPrecompile", "SparseArrays", "SuiteSparse"]
@@ -95,11 +99,6 @@ git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
version = "0.5.10"
-[[deps.BitFlags]]
-git-tree-sha1 = "43b1a4a8f797c1cddadf60499a8a077d4af2cd2d"
-uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35"
-version = "0.1.7"
-
[[deps.BufferedStreams]]
git-tree-sha1 = "bb065b14d7f941b8617bc323063dbe79f55d16ea"
uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
@@ -195,10 +194,10 @@ uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
version = "0.1.6"
[[deps.Chemfiles]]
-deps = ["BinaryProvider", "Chemfiles_jll", "DocStringExtensions", "Libdl"]
-git-tree-sha1 = "3b4a49a0a4c9b2ff8c0c6ec035c16f32955531a8"
+deps = ["Chemfiles_jll", "DocStringExtensions"]
+git-tree-sha1 = "9126d0271c337ca5ed02ba92f2dec087c4260d4a"
uuid = "46823bd8-5fb3-5f92-9aa0-96921f3dd015"
-version = "0.10.3"
+version = "0.10.31"
[[deps.Chemfiles_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -208,9 +207,15 @@ version = "0.10.3+0"
[[deps.Clustering]]
deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"]
-git-tree-sha1 = "64df3da1d2a26f4de23871cd1b6482bb68092bd5"
+git-tree-sha1 = "7ebbd653f74504447f1c33b91cd706a69a1b189f"
uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5"
-version = "0.14.3"
+version = "0.14.4"
+
+[[deps.CodeTracking]]
+deps = ["InteractiveUtils", "UUIDs"]
+git-tree-sha1 = "d57c99cc7e637165c81b30eb268eabe156a45c49"
+uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
+version = "1.2.2"
[[deps.CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
@@ -262,7 +267,6 @@ version = "4.6.1"
[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.0.1+0"
[[deps.CompositionsBase]]
git-tree-sha1 = "455419f7e328a1a2493cabc6428d79e951349769"
@@ -314,10 +318,10 @@ uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.14.0"
[[deps.DataDeps]]
-deps = ["HTTP", "Libdl", "Reexport", "SHA", "p7zip_jll"]
-git-tree-sha1 = "bc0a264d3e7b3eeb0b6fc9f6481f970697f29805"
+deps = ["BinaryProvider", "HTTP", "Libdl", "Reexport", "SHA", "p7zip_jll"]
+git-tree-sha1 = "e299d8267135ef2f9c941a764006697082c1e7e8"
uuid = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
-version = "0.7.10"
+version = "0.7.8"
[[deps.DataFrames]]
deps = ["Compat", "DataAPI", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SnoopPrecompile", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
@@ -396,9 +400,8 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.9.3"
[[deps.Downloads]]
-deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-version = "1.6.0"
[[deps.DualNumbers]]
deps = ["Calculus", "NaNMath", "SpecialFunctions"]
@@ -465,14 +468,11 @@ git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12"
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
version = "0.9.20"
-[[deps.FileWatching]]
-uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
-
[[deps.FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
-git-tree-sha1 = "9dec0199898d4d5c1d1b257cbf2cc498afe03a2a"
+git-tree-sha1 = "3b245d1e50466ca0c9529e2033a3c92387c59c2f"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.13.8"
+version = "0.13.9"
[[deps.FiniteDiff]]
deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays", "StaticArrays"]
@@ -551,9 +551,9 @@ version = "3.3.8+0"
[[deps.GPUArrays]]
deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"]
-git-tree-sha1 = "a28f752ffab0ccd6660fc7af5ad1c9ad176f45f7"
+git-tree-sha1 = "7a2e790b1e2e6f648cfb25c4500c5de1f7b375ef"
uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
-version = "8.6.3"
+version = "8.6.5"
[[deps.GPUArraysCore]]
deps = ["Adapt"]
@@ -569,15 +569,15 @@ version = "0.17.3"
[[deps.GR]]
deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Preferences", "Printf", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "UUIDs", "p7zip_jll"]
-git-tree-sha1 = "660b2ea2ec2b010bb02823c6d0ff6afd9bdc5c16"
+git-tree-sha1 = "4423d87dc2d3201f3f1768a29e807ddc8cc867ef"
uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
-version = "0.71.7"
+version = "0.71.8"
[[deps.GR_jll]]
deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "d5e1fd17ac7f3aa4c5287a61ee28d4f8b8e98873"
+git-tree-sha1 = "3657eb348d44575cc5560c80d7e55b812ff6ffe1"
uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
-version = "0.71.7+0"
+version = "0.71.8+0"
[[deps.GZip]]
deps = ["Libdl"]
@@ -632,10 +632,10 @@ uuid = "0234f1f7-429e-5d53-9886-15a909be8d59"
version = "1.12.2+2"
[[deps.HTTP]]
-deps = ["Base64", "CodecZlib", "Dates", "IniFile", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
-git-tree-sha1 = "37e4657cd56b11abe3d10cd4a1ec5fbdb4180263"
+deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
+git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "1.7.4"
+version = "0.9.17"
[[deps.HarfBuzz_jll]]
deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
@@ -643,6 +643,12 @@ git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3"
uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
version = "2.8.1+1"
+[[deps.Highlights]]
+deps = ["DocStringExtensions", "InteractiveUtils", "REPL"]
+git-tree-sha1 = "0341077e8a6b9fc1c2ea5edc1e93a956d2aec0c7"
+uuid = "eafb193a-b7ab-5a9e-9068-77385905fa72"
+version = "0.5.2"
+
[[deps.HypergeometricFunctions]]
deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions", "Test"]
git-tree-sha1 = "709d864e3ed6e3545230601f94e11ebc65994641"
@@ -810,15 +816,15 @@ version = "3.0.0+1"
[[deps.LLVM]]
deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"]
-git-tree-sha1 = "df115c31f5c163697eede495918d8e85045c8f04"
+git-tree-sha1 = "1c614dfbecbaee4897b506bba2b432bf0d21f2ed"
uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
-version = "4.16.0"
+version = "4.17.0"
[[deps.LLVMExtra_jll]]
-deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg", "TOML"]
-git-tree-sha1 = "7718cf44439c676bc0ec66a87099f41015a522d6"
+deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"]
+git-tree-sha1 = "e46e3a40daddcbe851f86db0ec4a4a3d4badf800"
uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
-version = "0.0.16+2"
+version = "0.0.19+0"
[[deps.LZO_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -855,12 +861,10 @@ version = "0.3.1"
[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.6.3"
[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
-version = "7.84.0+0"
[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
@@ -869,7 +873,6 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
-version = "1.10.2+0"
[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -929,7 +932,7 @@ uuid = "093fc24a-ae57-5d10-9952-331d41423f4d"
version = "1.3.5"
[[deps.LinearAlgebra]]
-deps = ["Libdl", "libblastrampoline_jll"]
+deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
[[deps.LogExpFunctions]]
@@ -941,12 +944,6 @@ version = "0.3.23"
[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
-[[deps.LoggingExtras]]
-deps = ["Dates", "Logging"]
-git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e"
-uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
-version = "1.0.0"
-
[[deps.LossFunctions]]
deps = ["InteractiveUtils", "Markdown", "RecipesBase"]
git-tree-sha1 = "53cd63a12f06a43eef6f4aafb910ac755c122be7"
@@ -961,9 +958,9 @@ version = "0.12.1"
[[deps.MAT]]
deps = ["BufferedStreams", "CodecZlib", "HDF5", "SparseArrays"]
-git-tree-sha1 = "971be550166fe3f604d28715302b58a3f7293160"
+git-tree-sha1 = "6eff5740c8ab02c90065719579c7aa0eb40c9f69"
uuid = "23992714-dd62-5051-b70f-ba57cb901cac"
-version = "0.10.3"
+version = "0.10.4"
[[deps.MKL_jll]]
deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"]
@@ -1042,7 +1039,6 @@ version = "1.1.7"
[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.0+0"
[[deps.Measures]]
git-tree-sha1 = "c13304c81eec1ed3af7fc20e75fb6b26092a1102"
@@ -1078,7 +1074,6 @@ version = "0.3.4"
[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2022.2.1"
[[deps.MultivariateStats]]
deps = ["Arpack", "LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI", "StatsBase"]
@@ -1086,6 +1081,11 @@ git-tree-sha1 = "91a48569383df24f0fd2baf789df2aade3d0ad80"
uuid = "6f286f6a-111f-5878-ab1e-185364afe411"
version = "0.10.1"
+[[deps.MyterialColors]]
+git-tree-sha1 = "01d8466fb449436348999d7c6ad740f8f853a579"
+uuid = "1c23619d-4212-4747-83aa-717207fae70f"
+version = "0.3.0"
+
[[deps.NLSolversBase]]
deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"]
git-tree-sha1 = "a0b464d183da839699f4c79e7606d9d186ec172c"
@@ -1147,7 +1147,6 @@ version = "0.4.13"
[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
-version = "1.2.0"
[[deps.Observables]]
git-tree-sha1 = "6862738f9796b3edc1c09d0890afce4eca9e7e93"
@@ -1175,18 +1174,10 @@ version = "0.2.3"
[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.20+0"
[[deps.OpenLibm_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
-version = "0.8.1+0"
-
-[[deps.OpenSSL]]
-deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"]
-git-tree-sha1 = "6503b77492fd7fcb9379bf73cd31035670e3c509"
-uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
-version = "1.3.3"
[[deps.OpenSSL_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -1226,7 +1217,6 @@ version = "1.4.1"
[[deps.PCRE2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15"
-version = "10.40.0+0"
[[deps.PDMats]]
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
@@ -1272,7 +1262,6 @@ version = "0.40.1+0"
[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.8.0"
[[deps.PlotThemes]]
deps = ["PlotUtils", "Statistics"]
@@ -1353,7 +1342,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
[[deps.Random]]
-deps = ["SHA", "Serialization"]
+deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
[[deps.Random123]]
@@ -1423,7 +1412,6 @@ version = "0.4.0+0"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-version = "0.7.0"
[[deps.ScientificTypes]]
deps = ["CategoricalArrays", "ColorTypes", "Dates", "Distributions", "PrettyTables", "Reexport", "ScientificTypesBase", "StatisticalTraits", "Tables"]
@@ -1472,11 +1460,6 @@ git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
version = "1.0.3"
-[[deps.SimpleBufferStream]]
-git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"
-uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7"
-version = "1.1.0"
-
[[deps.SimpleTraits]]
deps = ["InteractiveUtils", "MacroTools"]
git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231"
@@ -1607,7 +1590,6 @@ uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.0"
[[deps.TableOperations]]
deps = ["SentinelArrays", "Tables", "Test"]
@@ -1630,7 +1612,6 @@ version = "1.10.1"
[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
-version = "1.10.1"
[[deps.TensorCore]]
deps = ["LinearAlgebra"]
@@ -1638,6 +1619,12 @@ git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6"
uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50"
version = "0.1.1"
+[[deps.Term]]
+deps = ["AbstractTrees", "CodeTracking", "Dates", "Highlights", "InteractiveUtils", "Logging", "Markdown", "MyterialColors", "OrderedCollections", "Parameters", "ProgressLogging", "REPL", "SnoopPrecompile", "Tables", "UUIDs", "Unicode", "UnicodeFun"]
+git-tree-sha1 = "373d65207cb8de6d2e7bd32b89476e760c6edc4d"
+uuid = "22787eb5-b846-44ae-b979-8e399b8463ab"
+version = "2.0.2"
+
[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
@@ -1707,9 +1694,9 @@ uuid = "b8865327-cd53-5732-bb35-84acbb429228"
version = "3.4.1"
[[deps.Unzip]]
-git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78"
+git-tree-sha1 = "34db80951901073501137bdbc3d5a8e7bbd06670"
uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d"
-version = "0.2.0"
+version = "0.1.2"
[[deps.Wayland_jll]]
deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
@@ -1893,7 +1880,6 @@ version = "0.10.1"
[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.12+3"
[[deps.Zstd_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
@@ -1937,11 +1923,6 @@ git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
version = "0.15.1+0"
-[[deps.libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
-uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-version = "5.1.1+0"
-
[[deps.libfdk_aac_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
@@ -1963,12 +1944,10 @@ version = "1.3.7+1"
[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
-version = "1.48.0+0"
[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "17.4.0+0"
[[deps.x264_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
diff --git a/Project.toml b/Project.toml
index a6f587dfbc4eadb897077e1c0bc78777a9af114e..a47e33a222f873ece498fbdec84f915237d99f71 100644
--- a/Project.toml
+++ b/Project.toml
@@ -24,6 +24,7 @@ SliceMap = "82cb661a-3f19-5665-9e27-df437c7e54c8"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"
+Term = "22787eb5-b846-44ae-b979-8e399b8463ab"
[compat]
julia = "1.8"
diff --git a/_quarto.yml b/_quarto.yml
index 17737d0d1cd4663c80fecc5d06d81555f3e8e3a0..0f93a02a695f75384b8e73aac8a50146df77fd33 100644
--- a/_quarto.yml
+++ b/_quarto.yml
@@ -10,7 +10,7 @@ book:
date: today
chapters:
- index.qmd
- - notebooks/proposal.qmd
+ # - notebooks/proposal.qmd
- notebooks/intro.qmd
- notebooks/references.qmd
diff --git a/docs/index.html b/docs/index.html
index e26d982e597c3e069b6936959c6aaadedd11bf3d..c162a57282ec36e59cabb59c206ac27d08d53bed 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<meta name="author" content="Patrick Altmeyer">
-<meta name="dcterms.date" content="2023-03-20">
+<meta name="dcterms.date" content="2023-03-21">
<title>Conformal Counterfactual Explanations</title>
<style>
@@ -32,7 +32,7 @@ ul.task-list li input[type="checkbox"] {
<script src="site_libs/quarto-search/fuse.min.js"></script>
<script src="site_libs/quarto-search/quarto-search.js"></script>
<meta name="quarto:offset" content="./">
-<link href="./notebooks/proposal.html" rel="next">
+<link href="./notebooks/intro.html" rel="next">
<script src="site_libs/quarto-html/quarto.js"></script>
<script src="site_libs/quarto-html/popper.min.js"></script>
<script src="site_libs/quarto-html/tippy.umd.min.js"></script>
@@ -106,12 +106,7 @@ ul.task-list li input[type="checkbox"] {
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
- <a href="./notebooks/proposal.html" class="sidebar-item-text sidebar-link"><span class="chapter-number">1</span> <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="./notebooks/intro.html" class="sidebar-item-text sidebar-link"><span class="chapter-number">2</span> <span class="chapter-title"><code>ConformalGenerator</code></span></a>
+ <a href="./notebooks/intro.html" class="sidebar-item-text sidebar-link"><span class="chapter-number">1</span> <span class="chapter-title"><code>ConformalGenerator</code></span></a>
</div>
</li>
<li class="sidebar-item">
@@ -156,7 +151,7 @@ ul.task-list li input[type="checkbox"] {
<div>
<div class="quarto-title-meta-heading">Published</div>
<div class="quarto-title-meta-contents">
- <p class="date">March 20, 2023</p>
+ <p class="date">March 21, 2023</p>
</div>
</div>
@@ -396,8 +391,8 @@ window.document.addEventListener("DOMContentLoaded", function (event) {
<div class="nav-page nav-page-previous">
</div>
<div class="nav-page nav-page-next">
- <a href="./notebooks/proposal.html" class="pagination-link">
- <span class="nav-page-text"><span class="chapter-number">1</span> <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <i class="bi bi-arrow-right-short"></i>
+ <a href="./notebooks/intro.html" class="pagination-link">
+ <span class="nav-page-text"><span class="chapter-number">1</span> <span class="chapter-title"><code>ConformalGenerator</code></span></span> <i class="bi bi-arrow-right-short"></i>
</a>
</div>
</nav>
diff --git a/docs/notebooks/proposal.html b/docs/notebooks/proposal.html
deleted file mode 100644
index 68790b143f9160edc6ef5fd0158cbff69e67dc81..0000000000000000000000000000000000000000
--- a/docs/notebooks/proposal.html
+++ /dev/null
@@ -1,582 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
-
-<head>
-
-<meta charset="utf-8" />
-<meta name="generator" content="quarto-99.9.9" />
-
-<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
-
-
-<title>Conformal Counterfactual Explanations – 1 High-Fidelity Counterfactual Explanations through Conformal Prediction</title>
-<style>
-code{white-space: pre-wrap;}
-span.smallcaps{font-variant: small-caps;}
-div.columns{display: flex; gap: min(4vw, 1.5em);}
-div.column{flex: auto; overflow-x: auto;}
-div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
-ul.task-list{list-style: none;}
-ul.task-list li input[type="checkbox"] {
- width: 0.8em;
- margin: 0 0.8em 0.2em -1.6em;
- vertical-align: middle;
-}
-/* CSS for citations */
-div.csl-bib-body { }
-div.csl-entry {
- clear: both;
-}
-.hanging-indent div.csl-entry {
- margin-left:2em;
- text-indent:-2em;
-}
-div.csl-left-margin {
- min-width:2em;
- float:left;
-}
-div.csl-right-inline {
- margin-left:2em;
- padding-left:1em;
-}
-div.csl-indent {
- margin-left: 2em;
-}</style>
-
-<!-- htmldependencies:E3FAD763 -->
-<script id="quarto-search-options" type="application/json">{
- "location": "sidebar",
- "copy-button": false,
- "collapse-after": 3,
- "panel-placement": "start",
- "type": "textbox",
- "limit": 20,
- "language": {
- "search-no-results-text": "No results",
- "search-matching-documents-text": "matching documents",
- "search-copy-link-title": "Copy link to search",
- "search-hide-matches-text": "Hide additional matches",
- "search-more-match-text": "more match in this document",
- "search-more-matches-text": "more matches in this document",
- "search-clear-button-title": "Clear",
- "search-detached-cancel-button-title": "Cancel",
- "search-submit-button-title": "Submit"
- }
-}</script>
-
- <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>
-
-</head>
-
-<body>
-
-<div id="quarto-search-results"></div>
- <header id="quarto-header" class="headroom fixed-top">
- <nav class="quarto-secondary-nav">
- <div class="container-fluid d-flex">
- <button type="button" class="quarto-btn-toggle btn"
- data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass"
- aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation"
- onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
- <i class="bi bi-layout-text-sidebar-reverse"></i>
- </button>
- <h1 class="quarto-secondary-nav-title"></h1>
- <a class="flex-grow-1" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass"
- aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation"
- onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
- </a>
- <button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();">
- <i class="bi bi-search"></i>
- </button>
- </div>
- </nav>
-</header>
-<!-- content -->
-<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article">
-<!-- sidebar -->
- <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal sidebar-navigation floating overflow-auto">
- <div class="pt-lg-2 mt-2 text-left sidebar-header">
- <div class="sidebar-title mb-0 py-0">
- <a href="/">
- Conformal Counterfactual Explanations
- </a>
- </div>
- </div>
- <div class="mt-2 flex-shrink-0 align-items-center">
- <div class="sidebar-search">
- <div id="quarto-search" class="" title="Search"></div>
- </div>
- </div>
- <div class="sidebar-menu-container">
- <ul class="list-unstyled mt-1">
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/index.html" class="sidebar-item-text sidebar-link">Preface</a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/proposal.html" class="sidebar-item-text sidebar-link active"><span class='chapter-number'>1</span>Â <span class='chapter-title'>High-Fidelity Counterfactual Explanations through Conformal Prediction</span></a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/intro.html" class="sidebar-item-text sidebar-link"><span class='chapter-number'>2</span>Â <span class='chapter-title'>`ConformalGenerator`</span></a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/references.html" class="sidebar-item-text sidebar-link">References</a>
- </div>
-</li>
- </ul>
- </div>
-</nav>
-<div id="quarto-sidebar-glass" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass" ></div>
-<!-- margin-sidebar -->
- <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
- <div id="quarto-toc-target"></div>
- </div>
-<!-- main -->
-<main class="content" id="quarto-document-content">
-
-<header id="title-block-header" class="quarto-title-block default">
-<div class="quarto-title">
-<h1 class="title"><span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></h1>
-<p class="subtitle lead">Research Proposal</p>
-</div>
-
-
-
-<div class="quarto-title-meta">
-
-
-
-
- </div>
-
-<div>
- <div class="abstract">
- <div class="abstract-title">Abstract</div>
- <p>We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</p>
- </div>
-</div>
-
-</header>
-<nav id="TOC" role="doc-toc">
- <h2 id="toc-title">Table of contents</h2>
-
- <ul>
- <li><a href="#motivation" id="toc-motivation"><span class="header-section-number">1.1</span> Motivation</a>
- <ul>
- <li><a href="#counterfactual-explanations-or-adversarial-examples" id="toc-counterfactual-explanations-or-adversarial-examples"><span class="header-section-number">1.1.1</span> Counterfactual Explanations or Adversarial Examples?</a></li>
- <li><a href="#sec-fidelity" id="toc-sec-fidelity"><span class="header-section-number">1.1.2</span> From Plausible to High-Fidelity Counterfactuals</a></li>
- </ul></li>
- <li><a href="#conformal-counterfactual-explanations" id="toc-conformal-counterfactual-explanations"><span class="header-section-number">1.2</span> Conformal Counterfactual Explanations</a>
- <ul>
- <li><a href="#minimizing-predictive-uncertainty" id="toc-minimizing-predictive-uncertainty"><span class="header-section-number">1.2.1</span> Minimizing Predictive Uncertainty</a></li>
- <li><a href="#background-on-conformal-prediction" id="toc-background-on-conformal-prediction"><span class="header-section-number">1.2.2</span> Background on Conformal Prediction</a></li>
- <li><a href="#generating-conformal-counterfactuals" id="toc-generating-conformal-counterfactuals"><span class="header-section-number">1.2.3</span> Generating Conformal Counterfactuals</a></li>
- </ul></li>
- <li><a href="#experiments" id="toc-experiments"><span class="header-section-number">1.3</span> Experiments</a>
- <ul>
- <li><a href="#research-questions" id="toc-research-questions"><span class="header-section-number">1.3.1</span> Research Questions</a></li>
- </ul></li>
- <li><a href="#references" id="toc-references"><span class="header-section-number">1.4</span> References</a></li>
- </ul>
-</nav>
-<section id="motivation" class="level2" data-number="1.1">
-<h2 data-number="1.1"><span class="header-section-number">1.1</span> Motivation</h2>
-<p>Counterfactual Explanations are a powerful, flexible and intuitive way to not only explain black-box models but also enable affected individuals to challenge them through the means of Algorithmic Recourse.</p>
-<section id="counterfactual-explanations-or-adversarial-examples" class="level3" data-number="1.1.1">
-<h3 data-number="1.1.1"><span class="header-section-number">1.1.1</span> Counterfactual Explanations or Adversarial Examples?</h3>
-<p>Most state-of-the-art approaches to generating Counterfactual Explanations (CE) rely on gradient descent in the feature space. The key idea is to perturb inputs <span class="math inline">\(x\in\mathcal{X}\)</span> into a black-box model <span class="math inline">\(f: \mathcal{X} \mapsto \mathcal{Y}\)</span> in order to change the model output <span class="math inline">\(f(x)\)</span> to some pre-specified target value <span class="math inline">\(t\in\mathcal{Y}\)</span>. Formally, this boils down to defining some loss function <span class="math inline">\(\ell(f(x),t)\)</span> and taking gradient steps in the minimizing direction. The so-generated counterfactuals are considered valid as soon as the predicted label matches the target label. A stripped-down counterfactual explanation is therefore little different from an adversarial example. In <a href="#fig-adv">Figure <span class="quarto-unresolved-ref">fig-adv</span></a>, for example, generic counterfactual search as in <span class="citation" data-cites="wachter2017counterfactual">Wachter, Mittelstadt, and Russell (<a href="#ref-wachter2017counterfactual" role="doc-biblioref">2017</a>)</span> has been applied to MNIST data.</p>
-<div id="fig-adv" class="quarto-figure quarto-figure-center">
-<figure>
-<p><img src="www/you_may_not_like_it.png" class="img-fluid" /></p>
-<p><figcaption>Figure 1.1: You may not like it, but this is what stripped-down counterfactuals look like. Here we have used <span class="citation" data-cites="wachter2017counterfactual">Wachter, Mittelstadt, and Russell (<a href="#ref-wachter2017counterfactual" role="doc-biblioref">2017</a>)</span> to generate multiple counterfactuals for turning an 8 (eight) into a 3 (three).</figcaption></p>
-</figure>
-</div>
-<p>The crucial difference between adversarial examples and counterfactuals is one of intent. While adversarial examples are typically intended to go unnoticed, counterfactuals in the context of Explainable AI are generally sought to be “plausibleâ€, “realistic†or “feasibleâ€. To fulfil this latter goal, researchers have come up with a myriad of ways. <span class="citation" data-cites="joshi2019realistic">Joshi et al. (<a href="#ref-joshi2019realistic" role="doc-biblioref">2019</a>)</span> were among the first to suggest that instead of searching counterfactuals in the feature space, we can instead traverse a latent embedding learned by a surrogate generative model. Similarly, <span class="citation" data-cites="poyiadzi2020face">Poyiadzi et al. (<a href="#ref-poyiadzi2020face" role="doc-biblioref">2020</a>)</span> use density … Finally, <span class="citation" data-cites="karimi2021algorithmic">Karimi, Schölkopf, and Valera (<a href="#ref-karimi2021algorithmic" role="doc-biblioref">2021</a>)</span> argues that counterfactuals should comply with the causal model that generates them [CHECK IF WE CAN PHASE THIS LIKE THIS]. Other related approaches include … All of these different approaches have a common goal: they aim to ensure that the generated counterfactuals comply with the (learned) data-generating process (DGB).</p>
-<div id="def-plausible" class="theorem definition">
-<p><span class="theorem-title"><strong>Definition 1.1 (Plausible Counterfactuals) </strong></span>Formally, if <span class="math inline">\(x \sim \mathcal{X}\)</span> and for the corresponding counterfactual we have <span class="math inline">\(x^{\prime}\sim\mathcal{X}^{\prime}\)</span>, then for <span class="math inline">\(x^{\prime}\)</span> to be considered a plausible counterfactual, we need: <span class="math inline">\(\mathcal{X} \approxeq \mathcal{X}^{\prime}\)</span>.</p>
-</div>
-<p>In the context of Algorithmic Recourse, it makes sense to strive for plausible counterfactuals, since anything else would essentially require individuals to move to out-of-distribution states. But it is worth noting that our ambition to meet this goal, may have implications on our ability to faithfully explain the behaviour of the underlying black-box model (arguably our principal goal). By essentially decoupling the task of learning plausible representations of the data from the model itself, we open ourselves up to vulnerabilities. Using a separate generative model to learn <span class="math inline">\(\mathcal{X}\)</span>, for example, has very serious implications for the generated counterfactuals. <a href="#fig-latent">Figure <span class="quarto-unresolved-ref">fig-latent</span></a> compares the results of applying REVISE <span class="citation" data-cites="joshi2019realistic">(<a href="#ref-joshi2019realistic" role="doc-biblioref">Joshi et al. 2019</a>)</span> to MNIST data using two different Variational Auto-Encoders: while the counterfactual generated using an expressive (strong) VAE is compelling, the result relying on a less expressive (weak) VAE is not even valid. In this latter case, the decoder step of the VAE fails to yield values in <span class="math inline">\(\mathcal{X}\)</span> and hence the counterfactual search in the learned latent space is doomed.</p>
-<div id="fig-latent" class="quarto-figure quarto-figure-center">
-<figure>
-<p><img src="www/mnist_9to4_latent.png" class="img-fluid" /></p>
-<p><figcaption>Figure 1.2: Counterfactual explanations for MNIST using a Latent Space generator: turning a nine (9) into a four (4).</figcaption></p>
-</figure>
-</div>
-<blockquote>
-<p>Here it would be nice to have another example where we poison the data going into the generative model to hide biases present in the data (e.g. Boston housing).</p>
-</blockquote>
-<ul>
-<li>Latent can be manipulated:
-<ul>
-<li>train biased model</li>
-<li>train VAE with biased variable removed/attacked (use Boston housing dataset)</li>
-<li>hypothesis: will generate bias-free explanations</li>
-</ul></li>
-</ul>
-</section>
-<section id="sec-fidelity" class="level3" data-number="1.1.2">
-<h3 data-number="1.1.2"><span class="header-section-number">1.1.2</span> From Plausible to High-Fidelity Counterfactuals</h3>
-<p>In light of the findings, we propose to generally avoid using surrogate models to learn <span class="math inline">\(\mathcal{X}\)</span> in the context of Counterfactual Explanations.</p>
-<div id="prp-surrogate" class="theorem proposition">
-<p><span class="theorem-title"><strong>Proposition 1.1 (Avoid Surrogates) </strong></span>Since we are in the business of explaining a black-box model, the task of learning realistic representations of the data should not be reallocated from the model itself to some surrogate model.</p>
-</div>
-<p>In cases where the use of surrogate models cannot be avoided, we propose to weigh the plausibility of counterfactuals against their fidelity to the black-box model. In the context of Explainable AI, fidelity is defined as describing how an explanation approximates the prediction of the black-box model <span class="citation" data-cites="molnar2020interpretable">(<a href="#ref-molnar2020interpretable" role="doc-biblioref">Molnar 2020</a>)</span>. Fidelity has become the default metric for evaluating Local Model-Agnostic Models, since they often involve local surrogate models whose predictions need not always match those of the black-box model.</p>
-<p>In the case of Counterfactual Explanations, the concept of fidelity has so far been ignored. This is not altogether surprising, since by construction and design, Counterfactual Explanations work with the predictions of the black-box model directly: as stated above, a counterfactual <span class="math inline">\(x^{\prime}\)</span> is considered valid if and only if <span class="math inline">\(f(x^{\prime})=t\)</span>, where <span class="math inline">\(t\)</span> denote some target outcome.</p>
-<p>Does fidelity even make sense in the context of CE, and if so, how can we define it? In light of the examples in the previous section, we think it is urgent to introduce a notion of fidelity in this context, that relates to the distributional properties of the generated counterfactuals. In particular, we propose that a high-fidelity counterfactual <span class="math inline">\(x^{\prime}\)</span> complies with the class-conditional distribution <span class="math inline">\(\mathcal{X}_{\theta} = p_{\theta}(X|y)\)</span> where <span class="math inline">\(\theta\)</span> denote the black-box model parameters.</p>
-<div id="def-fidele" class="theorem definition">
-<p><span class="theorem-title"><strong>Definition 1.2 (High-Fidelity Counterfactuals) </strong></span>Let <span class="math inline">\(\mathcal{X}_{\theta}|y = p_{\theta}(X|y)\)</span> denote the class-conditional distribution of <span class="math inline">\(X\)</span> defined by <span class="math inline">\(\theta\)</span>. Then for <span class="math inline">\(x^{\prime}\)</span> to be considered a high-fidelity counterfactual, we need: <span class="math inline">\(\mathcal{X}_{\theta}|t \approxeq \mathcal{X}^{\prime}\)</span> where <span class="math inline">\(t\)</span> denotes the target outcome.</p>
-</div>
-<p>In order to assess the fidelity of counterfactuals, we propose the following two-step procedure:</p>
-<ol type="1">
-<li>Generate samples <span class="math inline">\(X_{\theta}|y\)</span> and <span class="math inline">\(X^{\prime}\)</span> from <span class="math inline">\(\mathcal{X}_{\theta}|t\)</span> and <span class="math inline">\(\mathcal{X}^{\prime}\)</span>, respectively.</li>
-<li>Compute the Maximum Mean Discrepancy (MMD) between <span class="math inline">\(X_{\theta}|y\)</span> and <span class="math inline">\(X^{\prime}\)</span>.</li>
-</ol>
-<p>If the computed value is different from zero, we can reject the null-hypothesis of fidelity.</p>
-<blockquote>
-<p>Two challenges here: 1) implementing the sampling procedure in <span class="citation" data-cites="grathwohl2020your">Grathwohl et al. (<a href="#ref-grathwohl2020your" role="doc-biblioref">2020</a>)</span>; 2) it is unclear if MMD is really the right way to measure this.</p>
-</blockquote>
-</section>
-</section>
-<section id="conformal-counterfactual-explanations" class="level2" data-number="1.2">
-<h2 data-number="1.2"><span class="header-section-number">1.2</span> Conformal Counterfactual Explanations</h2>
-<p>In <a href="#sec-fidelity"><span class="quarto-unresolved-ref">sec-fidelity</span></a>, we have advocated for avoiding surrogate models in the context of Counterfactual Explanations. In this section, we introduce an alternative way to generate high-fidelity Counterfactual Explanations. In particular, we propose Conformal Counterfactual Explanations (CCE), that is Counterfactual Explanations that minimize the predictive uncertainty of conformal models.</p>
-<section id="minimizing-predictive-uncertainty" class="level3" data-number="1.2.1">
-<h3 data-number="1.2.1"><span class="header-section-number">1.2.1</span> Minimizing Predictive Uncertainty</h3>
-<p><span class="citation" data-cites="schut2021generating">Schut et al. (<a href="#ref-schut2021generating" role="doc-biblioref">2021</a>)</span> demonstrated that the goal of generating realistic (plausible) counterfactuals can also be achieved by seeking counterfactuals that minimize the predictive uncertainty of the underlying black-box model. Similarly, <span class="citation" data-cites="antoran2020getting">Antorán et al. (<a href="#ref-antoran2020getting" role="doc-biblioref">2020</a>)</span> …</p>
-<ul>
-<li>Problem: restricted to Bayesian models.</li>
-<li>Solution: post-hoc predictive uncertainty quantification. In particular, Conformal Prediction.</li>
-</ul>
-</section>
-<section id="background-on-conformal-prediction" class="level3" data-number="1.2.2">
-<h3 data-number="1.2.2"><span class="header-section-number">1.2.2</span> Background on Conformal Prediction</h3>
-<ul>
-<li>Distribution-free, model-agnostic and scalable approach to predictive uncertainty quantification.</li>
-<li>Conformal prediction is instance-based. So is CE.</li>
-<li>Take any fitted model and turn it into a conformal model using calibration data.</li>
-<li>Our approach, therefore, relaxes the restriction on the family of black-box models, at the cost of relying on a subset of the data. Arguably, data is often abundant and in most applications practitioners tend to hold out a test data set anyway.</li>
-</ul>
-<blockquote>
-<p>Does the coverage guarantee carry over to counterfactuals?</p>
-</blockquote>
-</section>
-<section id="generating-conformal-counterfactuals" class="level3" data-number="1.2.3">
-<h3 data-number="1.2.3"><span class="header-section-number">1.2.3</span> Generating Conformal Counterfactuals</h3>
-<p>While Conformal Prediction has recently grown in popularity, it does introduce a challenge in the context of classification: the predictions of Conformal Classifiers are set-valued and therefore difficult to work with, since they are, for example, non-differentiable. Fortunately, <span class="citation" data-cites="stutz2022learning">Stutz et al. (<a href="#ref-stutz2022learning" role="doc-biblioref">2022</a>)</span> introduced carefully designed differentiable loss functions that make it possible to evaluate the performance of conformal predictions in training. We can leverage these recent advances in the context of gradient-based counterfactual search …</p>
-<blockquote>
-<p>Challenge: still need to implement these loss functions.</p>
-</blockquote>
-</section>
-</section>
-<section id="experiments" class="level2" data-number="1.3">
-<h2 data-number="1.3"><span class="header-section-number">1.3</span> Experiments</h2>
-<section id="research-questions" class="level3" data-number="1.3.1">
-<h3 data-number="1.3.1"><span class="header-section-number">1.3.1</span> Research Questions</h3>
-<ul>
-<li><p>Is CP alone enough to ensure realistic counterfactuals?</p></li>
-<li><p>Do counterfactuals improve further as the models get better?</p></li>
-<li><p>Do counterfactuals get more realistic as coverage</p></li>
-<li><p>What happens as we vary coverage and setsize?</p></li>
-<li><p>What happens as we improve the model robustness?</p></li>
-<li><p>What happens as we improve the model’s ability to incorporate predictive uncertainty (deep ensemble, laplace)?</p></li>
-<li><p>What happens if we combine with DiCE, ClaPROAR, Gravitational?</p></li>
-<li><p>What about CE robustness to endogenous shifts <span class="citation" data-cites="altmeyer2023endogenous">(<a href="#ref-altmeyer2023endogenous" role="doc-biblioref">Altmeyer et al. 2023</a>)</span>?</p></li>
-<li><p>Benchmarking:</p>
-<ul>
-<li>add PROBE <span class="citation" data-cites="pawelczyk2022probabilistically">(<a href="#ref-pawelczyk2022probabilistically" role="doc-biblioref">Pawelczyk et al. 2022</a>)</span> into the mix.</li>
-<li>compare travel costs to domain shits.</li>
-</ul></li>
-</ul>
-<blockquote>
-<p>Nice to have: What about using Laplace Approximation, then Conformal Prediction? What about using Conformalised Laplace?</p>
-</blockquote>
-</section>
-</section>
-<section id="references" class="level2" data-number="1.4">
-<h2 data-number="1.4"><span class="header-section-number">1.4</span> References</h2>
-<div id="quarto-navigation-envelope" class="hidden">
-<p><span class="hidden" data-render-id="quarto-int-sidebar-title">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-int-navbar-title">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-int-next"><span class="chapter-number">2</span>Â <span class="chapter-title"><code>ConformalGenerator</code></span></span> <span class="hidden" data-render-id="quarto-int-prev">Preface</span> <span class="hidden" data-render-id="quarto-int-sidebar:/index.html">Preface</span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/proposal.html"><span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/intro.html"><span class="chapter-number">2</span>Â <span class="chapter-title"><code>ConformalGenerator</code></span></span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/references.html">References</span></p>
-</div>
-<div id="quarto-meta-markdown" class="hidden">
-<p><span class="hidden" data-render-id="quarto-metatitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-twittercardtitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-ogcardtitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-metasitename">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-twittercarddesc">We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</span> <span class="hidden" data-render-id="quarto-ogcardddesc">We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</span></p>
-</div>
-<div id="refs" class="references csl-bib-body hanging-indent" role="list">
-<div id="ref-altmeyer2023endogenous" class="csl-entry" role="listitem">
-Altmeyer, Patrick, Giovan Angela, Aleksander Buszydlik, Karol Dobiczek, Arie van Deursen, and Cynthia Liem. 2023. <span>“Endogenous <span>Macrodynamics</span> in <span>Algorithmic</span> <span>Recourse</span>.â€</span> In <em>First <span>IEEE</span> <span>Conference</span> on <span>Secure</span> and <span>Trustworthy</span> <span>Machine</span> <span>Learning</span></em>.
-</div>
-<div id="ref-antoran2020getting" class="csl-entry" role="listitem">
-Antorán, Javier, Umang Bhatt, Tameem Adel, Adrian Weller, and José Miguel Hernández-Lobato. 2020. <span>“Getting a Clue: <span>A</span> Method for Explaining Uncertainty Estimates.â€</span> <a href="https://arxiv.org/abs/2006.06848">https://arxiv.org/abs/2006.06848</a>.
-</div>
-<div id="ref-grathwohl2020your" class="csl-entry" role="listitem">
-Grathwohl, Will, Kuan-Chieh Wang, Joern-Henrik Jacobsen, David Duvenaud, Mohammad Norouzi, and Kevin Swersky. 2020. <span>“Your Classifier Is Secretly an Energy Based Model and You Should Treat It Like One.â€</span> In. <a href="https://openreview.net/forum?id=Hkxzx0NtDB">https://openreview.net/forum?id=Hkxzx0NtDB</a>.
-</div>
-<div id="ref-joshi2019realistic" class="csl-entry" role="listitem">
-Joshi, Shalmali, Oluwasanmi Koyejo, Warut Vijitbenjaronk, Been Kim, and Joydeep Ghosh. 2019. <span>“Towards Realistic Individual Recourse and Actionable Explanations in Black-Box Decision Making Systems.â€</span> <a href="https://arxiv.org/abs/1907.09615">https://arxiv.org/abs/1907.09615</a>.
-</div>
-<div id="ref-karimi2021algorithmic" class="csl-entry" role="listitem">
-Karimi, Amir-Hossein, Bernhard Schölkopf, and Isabel Valera. 2021. <span>“Algorithmic Recourse: From Counterfactual Explanations to Interventions.â€</span> In <em>Proceedings of the 2021 <span>ACM Conference</span> on <span>Fairness</span>, <span>Accountability</span>, and <span>Transparency</span></em>, 353–62.
-</div>
-<div id="ref-molnar2020interpretable" class="csl-entry" role="listitem">
-Molnar, Christoph. 2020. <em>Interpretable Machine Learning</em>. <span>Lulu. com</span>.
-</div>
-<div id="ref-pawelczyk2022probabilistically" class="csl-entry" role="listitem">
-Pawelczyk, Martin, Teresa Datta, Johannes van-den-Heuvel, Gjergji Kasneci, and Himabindu Lakkaraju. 2022. <span>“Probabilistically <span>Robust</span> <span>Recourse</span>: <span>Navigating</span> the <span>Trade</span>-Offs Between <span>Costs</span> and <span>Robustness</span> in <span>Algorithmic</span> <span>Recourse</span>.â€</span> <em>arXiv Preprint arXiv:2203.06768</em>.
-</div>
-<div id="ref-poyiadzi2020face" class="csl-entry" role="listitem">
-Poyiadzi, Rafael, Kacper Sokol, Raul Santos-Rodriguez, Tijl De Bie, and Peter Flach. 2020. <span>“<span>FACE</span>: <span>Feasible</span> and Actionable Counterfactual Explanations.â€</span> In <em>Proceedings of the <span>AAAI</span>/<span>ACM Conference</span> on <span>AI</span>, <span>Ethics</span>, and <span>Society</span></em>, 344–50.
-</div>
-<div id="ref-schut2021generating" class="csl-entry" role="listitem">
-Schut, Lisa, Oscar Key, Rory Mc Grath, Luca Costabello, Bogdan Sacaleanu, Yarin Gal, et al. 2021. <span>“Generating <span>Interpretable Counterfactual Explanations By Implicit Minimisation</span> of <span>Epistemic</span> and <span>Aleatoric Uncertainties</span>.â€</span> In <em>International <span>Conference</span> on <span>Artificial Intelligence</span> and <span>Statistics</span></em>, 1756–64. <span>PMLR</span>.
-</div>
-<div id="ref-stutz2022learning" class="csl-entry" role="listitem">
-Stutz, David, Krishnamurthy Dj Dvijotham, Ali Taylan Cemgil, and Arnaud Doucet. 2022. <span>“Learning <span>Optimal</span> <span>Conformal</span> <span>Classifiers</span>.â€</span> In. <a href="https://openreview.net/forum?id=t8O-4LKFVx">https://openreview.net/forum?id=t8O-4LKFVx</a>.
-</div>
-<div id="ref-wachter2017counterfactual" class="csl-entry" role="listitem">
-Wachter, Sandra, Brent Mittelstadt, and Chris Russell. 2017. <span>“Counterfactual Explanations Without Opening the Black Box: <span>Automated</span> Decisions and the <span>GDPR</span>.â€</span> <em>Harv. JL & Tech.</em> 31: 841.
-</div>
-</div>
-</section>
-
-</main> <!-- /main -->
-<script id = "quarto-html-after-body" type="application/javascript">
-window.document.addEventListener("DOMContentLoaded", function (event) {
- const toggleBodyColorMode = (bsSheetEl) => {
- const mode = bsSheetEl.getAttribute("data-mode");
- const bodyEl = window.document.querySelector("body");
- if (mode === "dark") {
- bodyEl.classList.add("quarto-dark");
- bodyEl.classList.remove("quarto-light");
- } else {
- bodyEl.classList.add("quarto-light");
- bodyEl.classList.remove("quarto-dark");
- }
- }
- const toggleBodyColorPrimary = () => {
- const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
- if (bsSheetEl) {
- toggleBodyColorMode(bsSheetEl);
- }
- }
- toggleBodyColorPrimary();
- const icon = "";
- const anchorJS = new window.AnchorJS();
- anchorJS.options = {
- placement: 'right',
- icon: icon
- };
- anchorJS.add('.anchored');
- const clipboard = new window.ClipboardJS('.code-copy-button', {
- target: function(trigger) {
- return trigger.previousElementSibling;
- }
- });
- clipboard.on('success', function(e) {
- // button target
- const button = e.trigger;
- // don't keep focus
- button.blur();
- // flash "checked"
- button.classList.add('code-copy-button-checked');
- var currentTitle = button.getAttribute("title");
- button.setAttribute("title", "Copied!");
- let tooltip;
- if (window.bootstrap) {
- button.setAttribute("data-bs-toggle", "tooltip");
- button.setAttribute("data-bs-placement", "left");
- button.setAttribute("data-bs-title", "Copied!");
- tooltip = new bootstrap.Tooltip(button,
- { trigger: "manual",
- customClass: "code-copy-button-tooltip",
- offset: [0, -8]});
- tooltip.show();
- }
- setTimeout(function() {
- if (tooltip) {
- tooltip.hide();
- button.removeAttribute("data-bs-title");
- button.removeAttribute("data-bs-toggle");
- button.removeAttribute("data-bs-placement");
- }
- button.setAttribute("title", currentTitle);
- button.classList.remove('code-copy-button-checked');
- }, 1000);
- // clear code selection
- e.clearSelection();
- });
- function tippyHover(el, contentFn) {
- const config = {
- allowHTML: true,
- content: contentFn,
- maxWidth: 500,
- delay: 100,
- arrow: false,
- appendTo: function(el) {
- return el.parentElement;
- },
- interactive: true,
- interactiveBorder: 10,
- theme: 'quarto',
- placement: 'bottom-start'
- };
- window.tippy(el, config);
- }
- const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
- for (var i=0; i<noterefs.length; i++) {
- const ref = noterefs[i];
- tippyHover(ref, function() {
- // use id or data attribute instead here
- let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
- try { href = new URL(href).hash; } catch {}
- const id = href.replace(/^#\/?/, "");
- const note = window.document.getElementById(id);
- return note.innerHTML;
- });
- }
- let selectedAnnoteEl;
- const selectorForAnnotation = ( cell, annotation) => {
- let cellAttr = 'data-code-cell="' + cell + '"';
- let lineAttr = 'data-code-annotation="' + annotation + '"';
- const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
- return selector;
- }
- const selectCodeLines = (annoteEl) => {
- const doc = window.document;
- const targetCell = annoteEl.getAttribute("data-target-cell");
- const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
- const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
- const lines = annoteSpan.getAttribute("data-code-lines").split(",");
- const lineIds = lines.map((line) => {
- return targetCell + "-" + line;
- })
- let top = null;
- let height = null;
- let parent = null;
- if (lineIds.length > 0) {
- //compute the position of the single el (top and bottom and make a div)
- const el = window.document.getElementById(lineIds[0]);
- top = el.offsetTop;
- height = el.offsetHeight;
- parent = el.parentElement.parentElement;
- if (lineIds.length > 1) {
- const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
- const bottom = lastEl.offsetTop + lastEl.offsetHeight;
- height = bottom - top;
- }
- if (top !== null && height !== null && parent !== null) {
- // cook up a div (if necessary) and position it
- let div = window.document.getElementById("code-annotation-line-highlight");
- if (div === null) {
- div = window.document.createElement("div");
- div.setAttribute("id", "code-annotation-line-highlight");
- div.style.position = 'absolute';
- parent.appendChild(div);
- }
- div.style.top = top - 2 + "px";
- div.style.height = height + 4 + "px";
- let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
- if (gutterDiv === null) {
- gutterDiv = window.document.createElement("div");
- gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
- gutterDiv.style.position = 'absolute';
- const codeCell = window.document.getElementById(targetCell);
- const gutter = codeCell.querySelector('.code-annotation-gutter');
- gutter.appendChild(gutterDiv);
- }
- gutterDiv.style.top = top - 2 + "px";
- gutterDiv.style.height = height + 4 + "px";
- }
- selectedAnnoteEl = annoteEl;
- }
- };
- const unselectCodeLines = () => {
- const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
- elementsIds.forEach((elId) => {
- const div = window.document.getElementById(elId);
- if (div) {
- div.remove();
- }
- });
- selectedAnnoteEl = undefined;
- };
- // Attach click handler to the DT
- const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
- for (const annoteDlNode of annoteDls) {
- annoteDlNode.addEventListener('click', (event) => {
- const clickedEl = event.target;
- if (clickedEl !== selectedAnnoteEl) {
- unselectCodeLines();
- const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
- if (activeEl) {
- activeEl.classList.remove('code-annotation-active');
- }
- selectCodeLines(clickedEl);
- clickedEl.classList.add('code-annotation-active');
- } else {
- // Unselect the line
- unselectCodeLines();
- clickedEl.classList.remove('code-annotation-active');
- }
- });
- }
- const findCites = (el) => {
- const parentEl = el.parentElement;
- if (parentEl) {
- const cites = parentEl.dataset.cites;
- if (cites) {
- return {
- el,
- cites: cites.split(' ')
- };
- } else {
- return findCites(el.parentElement)
- }
- } else {
- return undefined;
- }
- };
- var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
- for (var i=0; i<bibliorefs.length; i++) {
- const ref = bibliorefs[i];
- const citeInfo = findCites(ref);
- if (citeInfo) {
- tippyHover(citeInfo.el, function() {
- var popup = window.document.createElement('div');
- citeInfo.cites.forEach(function(cite) {
- var citeDiv = window.document.createElement('div');
- citeDiv.classList.add('hanging-indent');
- citeDiv.classList.add('csl-entry');
- var biblioDiv = window.document.getElementById('ref-' + cite);
- if (biblioDiv) {
- citeDiv.innerHTML = biblioDiv.innerHTML;
- }
- popup.appendChild(citeDiv);
- });
- return popup.innerHTML;
- });
- }
- }
-});
-</script>
-<nav class="page-navigation">
- <div class="nav-page nav-page-previous">
- <a href="/index.html" class="pagination-link">
- <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text">Preface</span>
- </a>
- </div>
- <div class="nav-page nav-page-next">
- <a href="/notebooks/intro.html" class="pagination-link">
- <span class="nav-page-text"><span class='chapter-number'>2</span>Â <span class='chapter-title'>`ConformalGenerator`</span></span> <i class="bi bi-arrow-right-short"></i>
- </a>
- </div>
-</nav>
-</div> <!-- /content -->
-
-</body>
-
-</html>
\ No newline at end of file
diff --git a/notebooks/intro.ipynb b/notebooks/intro.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..91b560a6c714c6e89c7d815cb8220c28f740571c
--- /dev/null
+++ b/notebooks/intro.ipynb
@@ -0,0 +1,486 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "include(\"notebooks/setup.jl\")\n",
+ "eval(setup_notebooks)"
+ ],
+ "id": "dfd65706",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# `ConformalGenerator`\n",
+ "\n",
+ "In this section, we will look at a simple example involving synthetic data, a black-box model and a generic Conformal Counterfactual Generator.\n",
+ "\n",
+ "## Black-box Model\n",
+ "\n",
+ "We consider a simple binary classification problem. Let $(X_i, Y_i), \\ i=1,...,n$ denote our feature-label pairs and let $\\mu: \\mathcal{X} \\mapsto \\mathcal{Y}$ denote the mapping from features to labels. For illustration purposes, we will use linearly separable data. \n"
+ ],
+ "id": "4c30df47"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "counterfactual_data = load_linearly_separable()"
+ ],
+ "id": "b23030b1",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "While we could use a linear classifier in this case, let's pretend we need a black-box model for this task and rely on a small Multi-Layer Perceptron (MLP):\n"
+ ],
+ "id": "c95f7897"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "builder = MLJFlux.@builder Chain(\n",
+ " Dense(n_in, 32, relu),\n",
+ " Dense(32, n_out)\n",
+ ")\n",
+ "clf = NeuralNetworkClassifier(builder=builder, epochs=100)"
+ ],
+ "id": "aa6bb367",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can fit this model to data to produce plug-in predictions. \n",
+ "\n",
+ "## Conformal Prediction\n",
+ "\n",
+ "Here we will instead use a specific case of CP called *split conformal prediction* which can then be summarized as follows:^[In other places split conformal prediction is sometimes referred to as *inductive* conformal prediction.]\n",
+ "\n",
+ "1. Partition the training into a proper training set and a separate calibration set: $\\mathcal{D}_n=\\mathcal{D}^{\\text{train}} \\cup \\mathcal{D}^{\\text{cali}}$.\n",
+ "2. Train the machine learning model on the proper training set: $\\hat\\mu_{i \\in \\mathcal{D}^{\\text{train}}}(X_i,Y_i)$.\n",
+ "\n",
+ "The model $\\hat\\mu_{i \\in \\mathcal{D}^{\\text{train}}}$ can now produce plug-in predictions. \n",
+ "\n",
+ "::: callout-note\n",
+ "\n",
+ "## Starting Point\n",
+ "\n",
+ "Note that this represents the starting point in applications of Algorithmic Recourse: we have some pre-trained classifier $M$ for which we would like to generate plausible Counterfactual Explanations. Next, we turn to the calibration step. \n",
+ ":::\n",
+ "\n",
+ "3. Compute nonconformity scores, $\\mathcal{S}$, using the calibration data $\\mathcal{D}^{\\text{cali}}$ and the fitted model $\\hat\\mu_{i \\in \\mathcal{D}^{\\text{train}}}$. \n",
+ "4. For a user-specified desired coverage ratio $(1-\\alpha)$ compute the corresponding quantile, $\\hat{q}$, of the empirical distribution of nonconformity scores, $\\mathcal{S}$.\n",
+ "5. For the given quantile and test sample $X_{\\text{test}}$, form the corresponding conformal prediction set: \n",
+ "\n",
+ "$$\n",
+ "C(X_{\\text{test}})=\\{y:s(X_{\\text{test}},y) \\le \\hat{q}\\}\n",
+ "$$ {#eq-set}\n",
+ "\n",
+ "This is the default procedure used for classification and regression in [`ConformalPrediction.jl`](https://github.com/pat-alt/ConformalPrediction.jl). \n",
+ "\n",
+ "Using the package, we can apply Split Conformal Prediction as follows:\n"
+ ],
+ "id": "56b7cbf3"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "X = table(permutedims(counterfactual_data.X))\n",
+ "y = counterfactual_data.output_encoder.labels\n",
+ "conf_model = conformal_model(clf; method=:simple_inductive)\n",
+ "mach = machine(conf_model, X, y)\n",
+ "fit!(mach)"
+ ],
+ "id": "508249d0",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "To be clear, all of the calibration steps (3 to 5) are post hoc, and yet none of them involved any changes to the model parameters. These are two important characteristics of Split Conformal Prediction (SCP) that make it particularly useful in the context of Algorithmic Recourse. Firstly, the fact that SCP involves posthoc calibration steps that happen after training, ensures that we need not place any restrictions on the black-box model itself. This stands in contrast to the approach proposed by @schut2021generating in which they essentially restrict the class of models to Bayesian models. Secondly, the fact that the model itself is kept entirely intact ensures that the generated counterfactuals maintain fidelity to the model. Finally, note that we also have not resorted to a surrogate model to learn more about $X \\sim \\mathcal{X}$. Instead, we have used the fitted model itself and a calibration data set to learn about the model's predictive uncertainty. \n",
+ "\n",
+ "## Differentiable CP\n",
+ "\n",
+ "In order to use CP in the context of gradient-based counterfactual search, we need it to be differentiable. @stutz2022learning introduce a framework for training differentiable conformal predictors. They introduce a configurable loss function as well as smooth set size penalty.\n",
+ "\n",
+ "### Smooth Set Size Penalty\n",
+ "\n",
+ "Starting with the former, @stutz2022learning propose the following:\n",
+ "\n",
+ "$$\n",
+ "\\Omega(C_{\\theta}(x;\\tau)) = = \\max (0, \\sum_k C_{\\theta,k}(x;\\tau) - \\kappa)\n",
+ "$$ {#eq-size-loss}\n",
+ "\n",
+ "Here, $C_{\\theta,k}(x;\\tau)$ is loosely defined as the probability that class $k$ is assigned to the conformal prediction set $C$. In the context of Conformal Training, this penalty reduces the **inefficiency** of the conformal predictor. \n",
+ "\n",
+ "In our context, we are not interested in improving the model itself, but rather in producing **plausible** counterfactuals. Provided that our counterfactual $x^\\prime$ is already inside the target domain ($\\mathbb{I}_{y^\\prime = t}=1$), penalizing $\\Omega(C_{\\theta}(x;\\tau))$ corresponds to guiding counterfactuals into regions of the target domain that are characterized by low ambiguity: for $\\kappa=1$ the conformal prediction set includes only the target label $t$ as $\\Omega(C_{\\theta}(x;\\tau))$. Arguably, less ambiguous counterfactuals are more **plausible**. Since the search is guided purely by properties of the model itself and (exchangeable) calibration data, counterfactuals also maintain **high fidelity**.\n",
+ "\n",
+ "The left panel of @fig-losses shows the smooth size penalty in the two-dimensional feature space of our synthetic data.\n",
+ "\n",
+ "### Configurable Classification Loss\n",
+ "\n",
+ "The right panel of @fig-losses shows the configurable classification loss in the two-dimensional feature space of our synthetic data.\n"
+ ],
+ "id": "89d53442"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| output: true\n",
+ "#| echo: false\n",
+ "#| label: fig-losses\n",
+ "#| fig-cap: Illustration of the smooth size loss and the configurable classification loss.\n",
+ "\n",
+ "temp = 0.5\n",
+ "p1 = contourf(mach.model, mach.fitresult, X, y; plot_set_loss=true, zoom=0, temp=temp)\n",
+ "p2 = contourf(mach.model, mach.fitresult, X, y; plot_classification_loss=true, target=1, zoom=0, temp=temp, clim=nothing, loss_matrix=ones(2,2))\n",
+ "plot(p1, p2, size=(800,320))"
+ ],
+ "id": "fig-losses",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Fidelity and Plausibility\n",
+ "\n",
+ "The main evaluation criteria we are interested in are *fidelity* and *plausibility*. Interestingly, we could also consider using these measures as penalties in the counterfactual search.\n",
+ "\n",
+ "### Fidelity\n",
+ "\n",
+ "We propose to define fidelity as follows:\n",
+ "\n",
+ "::: {#def-fidelity}\n",
+ "\n",
+ "## High-Fidelity Counterfactuals\n",
+ "\n",
+ "Let $\\mathcal{X}_{\\theta}|y = p_{\\theta}(X|y)$ denote the class-conditional distribution of $X$ defined by $\\theta$. Then for $x^{\\prime}$ to be considered a high-fidelity counterfactual, we need: $\\mathcal{X}_{\\theta}|t \\approxeq \\mathcal{X}^{\\prime}$ where $t$ denotes the target outcome.\n",
+ "\n",
+ ":::\n",
+ "\n",
+ "We can generate samples from $p_{\\theta}(X|y)$ following @grathwohl2020your. In @fig-energy, I have applied the methodology to our synthetic data.\n"
+ ],
+ "id": "a7329cea"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| fig-cap: Energy-based conditional samples.\n",
+ "#| label: fig-energy\n",
+ "\n",
+ "M = CCE.ConformalModel(conf_model, mach.fitresult)\n",
+ "\n",
+ "niter = 100\n",
+ "nsamples = 100\n",
+ "\n",
+ "plts = []\n",
+ "for target ∈ counterfactual_data.y_levels\n",
+ " sampler = CCE.EnergySampler(M, counterfactual_data, target; niter=niter, nsamples=100)\n",
+ " Xgen = rand(sampler, nsamples)\n",
+ " plt = plot(M, counterfactual_data; target=target, zoom=-3,cbar=false)\n",
+ " scatter!(Xgen[1,:],Xgen[2,:],alpha=0.5,color=target,shape=:star,label=\"X|y=$target\")\n",
+ " push!(plts, plt)\n",
+ "end\n",
+ "plot(plts..., layout=(1,length(plts)), size=(img_height*length(plts),img_height))"
+ ],
+ "id": "fig-energy",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As an evaluation metric and penalty, we could use the average distance of the counterfactual $x^{\\prime}$ from these generated samples, for example.\n",
+ "\n",
+ "### Plausibility\n",
+ "\n",
+ "We propose to define plausibility as follows:\n",
+ "\n",
+ "::: {#def-plausible}\n",
+ "\n",
+ "## Plausible Counterfactuals\n",
+ "\n",
+ "Formally, let $\\mathcal{X}|t$ denote the conditional distribution of samples in the target class. As before, we have $x^{\\prime}\\sim\\mathcal{X}^{\\prime}$, then for $x^{\\prime}$ to be considered a plausible counterfactual, we need: $\\mathcal{X}|t \\approxeq \\mathcal{X}^{\\prime}$.\n",
+ "\n",
+ ":::\n",
+ "\n",
+ "As an evaluation metric and penalty, we could use the average distance of the counterfactual $x^{\\prime}$ from (potentially bootstrapped) training samples in the target class, for example.\n",
+ "\n",
+ "## Counterfactual Explanations\n",
+ "\n",
+ "Next, let's generate counterfactual explanations for our synthetic data. We first wrap our model in a container that makes it compatible with `CounterfactualExplanations.jl`. Then we draw a random sample, determine its predicted label $\\hat{y}$ and choose the opposite label as our target. \n"
+ ],
+ "id": "167a8309"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "x = select_factual(counterfactual_data,rand(1:size(counterfactual_data.X,2)))\n",
+ "y_factual = predict_label(M, counterfactual_data, x)[1]\n",
+ "target = counterfactual_data.y_levels[counterfactual_data.y_levels .!= y_factual][1]"
+ ],
+ "id": "aa344e11",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The generic Conformal Counterfactual Generator penalises the only the set size only:\n",
+ "\n",
+ "$$\n",
+ "x^\\prime = \\arg \\min_{x^\\prime} \\ell(M(x^\\prime),t) + \\lambda \\mathbb{I}_{y^\\prime = t} \\Omega(C_{\\theta}(x;\\tau)) \n",
+ "$$ {#eq-solution}\n"
+ ],
+ "id": "6d705569"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| output: true\n",
+ "#| echo: false\n",
+ "#| label: fig-ce\n",
+ "#| fig-cap: Comparison of counterfactuals produced using different generators.\n",
+ "\n",
+ "λ = 10.0\n",
+ "\n",
+ "opt = Descent(0.01)\n",
+ "ordered_names = [\n",
+ " \"Distance x\",\n",
+ " \"Set Size\",\n",
+ " \"Distance X_θ|t\",\n",
+ " \"Distance X|t\"\n",
+ "]\n",
+ "loss_fun = Objectives.logitbinarycrossentropy\n",
+ "generator = GenericGenerator(opt = opt)\n",
+ "\n",
+ "# Generators:\n",
+ "generators = Dict(\n",
+ " ordered_names[1] => generator,\n",
+ " ordered_names[2] => deepcopy(generator) |> gen -> @objective(gen, _ + 0.1distance + 10.0set_size_penalty),\n",
+ " ordered_names[3] => deepcopy(generator) |> gen -> @objective(gen, _ + 0.1distance + 10.0distance_from_energy),\n",
+ " ordered_names[4] => deepcopy(generator) |> gen -> @objective(gen, _ + 0.1distance + 10.0distance_from_targets),\n",
+ ")\n",
+ "\n",
+ "counterfactuals = Dict([name => generate_counterfactual(x, target, counterfactual_data, M, gen; initialization=:identity, converge_when=:generator_conditions, gradient_tol=1e-3) for (name, gen) in generators])\n",
+ "\n",
+ "# Plots:\n",
+ "plts = []\n",
+ "for name ∈ ordered_names\n",
+ " ce = counterfactuals[name]\n",
+ " plt = plot(ce; title=name, colorbar=false, ticks = false, legend=false, zoom=-0.5)\n",
+ " if :energy_sampler ∈ collect(keys(ce.params))\n",
+ " Xgen = ce.params[:energy_sampler].buffer\n",
+ " scatter!(Xgen[1,:],Xgen[2,:],alpha=0.5,color=target,shape=:star,label=\"X|y=$target\")\n",
+ " end\n",
+ " plts = vcat(plts..., plt)\n",
+ "end\n",
+ "_n = length(generators)\n",
+ "plot(plts..., size=(_n * img_height,1.05*img_height), layout=(1,_n))"
+ ],
+ "id": "fig-ce",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Benchmark\n"
+ ],
+ "id": "23703549"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| output: false\n",
+ "\n",
+ "# Data:\n",
+ "datasets = Dict(\n",
+ " :linearly_separable => load_linearly_separable(),\n",
+ " :overlapping => load_overlapping(),\n",
+ " :moons => load_moons(),\n",
+ " :circles => load_circles(),\n",
+ ")\n",
+ "\n",
+ "# Models:\n",
+ "models = Dict(\n",
+ " :MLP => M,\n",
+ ")"
+ ],
+ "id": "60b7b6b1",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then we can simply loop over the datasets and eventually concatenate the results like so:\n"
+ ],
+ "id": "10d69d5f"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| output: false\n",
+ "\n",
+ "measure = \n",
+ "using CounterfactualExplanations.Evaluation: benchmark\n",
+ "bmks = []\n",
+ "for (dataname, dataset) in datasets\n",
+ " bmk = benchmark(dataset; models=models, generators=generators, suppress_training=true, data_name=dataname)\n",
+ " push!(bmks, bmk)\n",
+ "end\n",
+ "bmk = reduce(vcat, bmks)"
+ ],
+ "id": "370a4431",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Multi-Class\n"
+ ],
+ "id": "fcab2bf0"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "counterfactual_data = load_multi_class()"
+ ],
+ "id": "cd4e3272",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "X = table(permutedims(counterfactual_data.X))\n",
+ "y = counterfactual_data.output_encoder.labels\n",
+ "conf_model = conformal_model(clf; method=:simple_inductive)\n",
+ "mach = machine(conf_model, X, y)\n",
+ "fit!(mach)"
+ ],
+ "id": "38531347",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| output: true\n",
+ "#| echo: false\n",
+ "#| label: fig-losses-multi\n",
+ "#| fig-cap: Illustration of the smooth size loss and the configurable classification loss.\n",
+ "\n",
+ "temp = 0.5\n",
+ "p1 = contourf(mach.model, mach.fitresult, X, y; plot_set_loss=true, zoom=0, temp=temp)\n",
+ "p2 = contourf(mach.model, mach.fitresult, X, y; plot_classification_loss=true, target=1, zoom=0, temp=temp, clim=nothing, loss_matrix=ones(4,4))\n",
+ "plot(p1, p2, size=(800,320))"
+ ],
+ "id": "fig-losses-multi",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| fig-cap: Energy-based conditional samples.\n",
+ "#| label: fig-energy-multi\n",
+ "\n",
+ "niter = 100\n",
+ "nsamples = 100\n",
+ "M = CCE.ConformalModel(conf_model, mach.fitresult; likelihood=:classification_multi)\n",
+ "\n",
+ "plts = []\n",
+ "for target ∈ counterfactual_data.y_levels\n",
+ " sampler = CCE.EnergySampler(M, counterfactual_data, target; niter=niter, nsamples=100)\n",
+ " Xgen = rand(sampler, nsamples)\n",
+ " plt = plot(M, counterfactual_data; target=target, zoom=-0.5,cbar=false)\n",
+ " scatter!(Xgen[1,:],Xgen[2,:],alpha=0.5,color=target,shape=:star,label=\"X|y=$target\")\n",
+ " push!(plts, plt)\n",
+ "end\n",
+ "plot(plts..., layout=(1,length(plts)), size=(img_height*length(plts),img_height))"
+ ],
+ "id": "fig-energy-multi",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "x = select_factual(counterfactual_data,rand(1:size(counterfactual_data.X,2)))\n",
+ "y_factual = predict_label(M, counterfactual_data, x)[1]\n",
+ "target = counterfactual_data.y_levels[counterfactual_data.y_levels .!= y_factual][1]"
+ ],
+ "id": "aec30b5c",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "counterfactuals = Dict([name => generate_counterfactual(x, target, counterfactual_data, M, gen; initialization=:identity, converge_when=:generator_conditions, gradient_tol=1e-3) for (name, gen) in generators])\n",
+ "\n",
+ "# Plots:\n",
+ "plts = []\n",
+ "for name ∈ ordered_names\n",
+ " ce = counterfactuals[name]\n",
+ " plt = plot(ce; title=name, colorbar=false, ticks = false, legend=false, zoom=-0.5)\n",
+ " if :energy_sampler ∈ collect(keys(ce.params))\n",
+ " Xgen = ce.params[:energy_sampler].buffer\n",
+ " scatter!(Xgen[1,:],Xgen[2,:],alpha=0.5,color=target,shape=:star,label=\"X|y=$target\")\n",
+ " end\n",
+ " plts = vcat(plts..., plt)\n",
+ "end\n",
+ "_n = length(generators)\n",
+ "plot(plts..., size=(_n * img_height,1.05*img_height), layout=(1,_n))"
+ ],
+ "id": "7b7ee911",
+ "execution_count": null,
+ "outputs": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "name": "julia-1.8",
+ "language": "julia",
+ "display_name": "Julia 1.8.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
\ No newline at end of file
diff --git a/notebooks/proposal.html b/notebooks/proposal.html
deleted file mode 100644
index 68790b143f9160edc6ef5fd0158cbff69e67dc81..0000000000000000000000000000000000000000
--- a/notebooks/proposal.html
+++ /dev/null
@@ -1,582 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
-
-<head>
-
-<meta charset="utf-8" />
-<meta name="generator" content="quarto-99.9.9" />
-
-<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
-
-
-<title>Conformal Counterfactual Explanations – 1 High-Fidelity Counterfactual Explanations through Conformal Prediction</title>
-<style>
-code{white-space: pre-wrap;}
-span.smallcaps{font-variant: small-caps;}
-div.columns{display: flex; gap: min(4vw, 1.5em);}
-div.column{flex: auto; overflow-x: auto;}
-div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
-ul.task-list{list-style: none;}
-ul.task-list li input[type="checkbox"] {
- width: 0.8em;
- margin: 0 0.8em 0.2em -1.6em;
- vertical-align: middle;
-}
-/* CSS for citations */
-div.csl-bib-body { }
-div.csl-entry {
- clear: both;
-}
-.hanging-indent div.csl-entry {
- margin-left:2em;
- text-indent:-2em;
-}
-div.csl-left-margin {
- min-width:2em;
- float:left;
-}
-div.csl-right-inline {
- margin-left:2em;
- padding-left:1em;
-}
-div.csl-indent {
- margin-left: 2em;
-}</style>
-
-<!-- htmldependencies:E3FAD763 -->
-<script id="quarto-search-options" type="application/json">{
- "location": "sidebar",
- "copy-button": false,
- "collapse-after": 3,
- "panel-placement": "start",
- "type": "textbox",
- "limit": 20,
- "language": {
- "search-no-results-text": "No results",
- "search-matching-documents-text": "matching documents",
- "search-copy-link-title": "Copy link to search",
- "search-hide-matches-text": "Hide additional matches",
- "search-more-match-text": "more match in this document",
- "search-more-matches-text": "more matches in this document",
- "search-clear-button-title": "Clear",
- "search-detached-cancel-button-title": "Cancel",
- "search-submit-button-title": "Submit"
- }
-}</script>
-
- <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>
-
-</head>
-
-<body>
-
-<div id="quarto-search-results"></div>
- <header id="quarto-header" class="headroom fixed-top">
- <nav class="quarto-secondary-nav">
- <div class="container-fluid d-flex">
- <button type="button" class="quarto-btn-toggle btn"
- data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass"
- aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation"
- onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
- <i class="bi bi-layout-text-sidebar-reverse"></i>
- </button>
- <h1 class="quarto-secondary-nav-title"></h1>
- <a class="flex-grow-1" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass"
- aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation"
- onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
- </a>
- <button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();">
- <i class="bi bi-search"></i>
- </button>
- </div>
- </nav>
-</header>
-<!-- content -->
-<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article">
-<!-- sidebar -->
- <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal sidebar-navigation floating overflow-auto">
- <div class="pt-lg-2 mt-2 text-left sidebar-header">
- <div class="sidebar-title mb-0 py-0">
- <a href="/">
- Conformal Counterfactual Explanations
- </a>
- </div>
- </div>
- <div class="mt-2 flex-shrink-0 align-items-center">
- <div class="sidebar-search">
- <div id="quarto-search" class="" title="Search"></div>
- </div>
- </div>
- <div class="sidebar-menu-container">
- <ul class="list-unstyled mt-1">
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/index.html" class="sidebar-item-text sidebar-link">Preface</a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/proposal.html" class="sidebar-item-text sidebar-link active"><span class='chapter-number'>1</span>Â <span class='chapter-title'>High-Fidelity Counterfactual Explanations through Conformal Prediction</span></a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/intro.html" class="sidebar-item-text sidebar-link"><span class='chapter-number'>2</span>Â <span class='chapter-title'>`ConformalGenerator`</span></a>
- </div>
-</li>
- <li class="sidebar-item">
- <div class="sidebar-item-container">
- <a href="/notebooks/references.html" class="sidebar-item-text sidebar-link">References</a>
- </div>
-</li>
- </ul>
- </div>
-</nav>
-<div id="quarto-sidebar-glass" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass" ></div>
-<!-- margin-sidebar -->
- <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
- <div id="quarto-toc-target"></div>
- </div>
-<!-- main -->
-<main class="content" id="quarto-document-content">
-
-<header id="title-block-header" class="quarto-title-block default">
-<div class="quarto-title">
-<h1 class="title"><span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></h1>
-<p class="subtitle lead">Research Proposal</p>
-</div>
-
-
-
-<div class="quarto-title-meta">
-
-
-
-
- </div>
-
-<div>
- <div class="abstract">
- <div class="abstract-title">Abstract</div>
- <p>We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</p>
- </div>
-</div>
-
-</header>
-<nav id="TOC" role="doc-toc">
- <h2 id="toc-title">Table of contents</h2>
-
- <ul>
- <li><a href="#motivation" id="toc-motivation"><span class="header-section-number">1.1</span> Motivation</a>
- <ul>
- <li><a href="#counterfactual-explanations-or-adversarial-examples" id="toc-counterfactual-explanations-or-adversarial-examples"><span class="header-section-number">1.1.1</span> Counterfactual Explanations or Adversarial Examples?</a></li>
- <li><a href="#sec-fidelity" id="toc-sec-fidelity"><span class="header-section-number">1.1.2</span> From Plausible to High-Fidelity Counterfactuals</a></li>
- </ul></li>
- <li><a href="#conformal-counterfactual-explanations" id="toc-conformal-counterfactual-explanations"><span class="header-section-number">1.2</span> Conformal Counterfactual Explanations</a>
- <ul>
- <li><a href="#minimizing-predictive-uncertainty" id="toc-minimizing-predictive-uncertainty"><span class="header-section-number">1.2.1</span> Minimizing Predictive Uncertainty</a></li>
- <li><a href="#background-on-conformal-prediction" id="toc-background-on-conformal-prediction"><span class="header-section-number">1.2.2</span> Background on Conformal Prediction</a></li>
- <li><a href="#generating-conformal-counterfactuals" id="toc-generating-conformal-counterfactuals"><span class="header-section-number">1.2.3</span> Generating Conformal Counterfactuals</a></li>
- </ul></li>
- <li><a href="#experiments" id="toc-experiments"><span class="header-section-number">1.3</span> Experiments</a>
- <ul>
- <li><a href="#research-questions" id="toc-research-questions"><span class="header-section-number">1.3.1</span> Research Questions</a></li>
- </ul></li>
- <li><a href="#references" id="toc-references"><span class="header-section-number">1.4</span> References</a></li>
- </ul>
-</nav>
-<section id="motivation" class="level2" data-number="1.1">
-<h2 data-number="1.1"><span class="header-section-number">1.1</span> Motivation</h2>
-<p>Counterfactual Explanations are a powerful, flexible and intuitive way to not only explain black-box models but also enable affected individuals to challenge them through the means of Algorithmic Recourse.</p>
-<section id="counterfactual-explanations-or-adversarial-examples" class="level3" data-number="1.1.1">
-<h3 data-number="1.1.1"><span class="header-section-number">1.1.1</span> Counterfactual Explanations or Adversarial Examples?</h3>
-<p>Most state-of-the-art approaches to generating Counterfactual Explanations (CE) rely on gradient descent in the feature space. The key idea is to perturb inputs <span class="math inline">\(x\in\mathcal{X}\)</span> into a black-box model <span class="math inline">\(f: \mathcal{X} \mapsto \mathcal{Y}\)</span> in order to change the model output <span class="math inline">\(f(x)\)</span> to some pre-specified target value <span class="math inline">\(t\in\mathcal{Y}\)</span>. Formally, this boils down to defining some loss function <span class="math inline">\(\ell(f(x),t)\)</span> and taking gradient steps in the minimizing direction. The so-generated counterfactuals are considered valid as soon as the predicted label matches the target label. A stripped-down counterfactual explanation is therefore little different from an adversarial example. In <a href="#fig-adv">Figure <span class="quarto-unresolved-ref">fig-adv</span></a>, for example, generic counterfactual search as in <span class="citation" data-cites="wachter2017counterfactual">Wachter, Mittelstadt, and Russell (<a href="#ref-wachter2017counterfactual" role="doc-biblioref">2017</a>)</span> has been applied to MNIST data.</p>
-<div id="fig-adv" class="quarto-figure quarto-figure-center">
-<figure>
-<p><img src="www/you_may_not_like_it.png" class="img-fluid" /></p>
-<p><figcaption>Figure 1.1: You may not like it, but this is what stripped-down counterfactuals look like. Here we have used <span class="citation" data-cites="wachter2017counterfactual">Wachter, Mittelstadt, and Russell (<a href="#ref-wachter2017counterfactual" role="doc-biblioref">2017</a>)</span> to generate multiple counterfactuals for turning an 8 (eight) into a 3 (three).</figcaption></p>
-</figure>
-</div>
-<p>The crucial difference between adversarial examples and counterfactuals is one of intent. While adversarial examples are typically intended to go unnoticed, counterfactuals in the context of Explainable AI are generally sought to be “plausibleâ€, “realistic†or “feasibleâ€. To fulfil this latter goal, researchers have come up with a myriad of ways. <span class="citation" data-cites="joshi2019realistic">Joshi et al. (<a href="#ref-joshi2019realistic" role="doc-biblioref">2019</a>)</span> were among the first to suggest that instead of searching counterfactuals in the feature space, we can instead traverse a latent embedding learned by a surrogate generative model. Similarly, <span class="citation" data-cites="poyiadzi2020face">Poyiadzi et al. (<a href="#ref-poyiadzi2020face" role="doc-biblioref">2020</a>)</span> use density … Finally, <span class="citation" data-cites="karimi2021algorithmic">Karimi, Schölkopf, and Valera (<a href="#ref-karimi2021algorithmic" role="doc-biblioref">2021</a>)</span> argues that counterfactuals should comply with the causal model that generates them [CHECK IF WE CAN PHASE THIS LIKE THIS]. Other related approaches include … All of these different approaches have a common goal: they aim to ensure that the generated counterfactuals comply with the (learned) data-generating process (DGB).</p>
-<div id="def-plausible" class="theorem definition">
-<p><span class="theorem-title"><strong>Definition 1.1 (Plausible Counterfactuals) </strong></span>Formally, if <span class="math inline">\(x \sim \mathcal{X}\)</span> and for the corresponding counterfactual we have <span class="math inline">\(x^{\prime}\sim\mathcal{X}^{\prime}\)</span>, then for <span class="math inline">\(x^{\prime}\)</span> to be considered a plausible counterfactual, we need: <span class="math inline">\(\mathcal{X} \approxeq \mathcal{X}^{\prime}\)</span>.</p>
-</div>
-<p>In the context of Algorithmic Recourse, it makes sense to strive for plausible counterfactuals, since anything else would essentially require individuals to move to out-of-distribution states. But it is worth noting that our ambition to meet this goal, may have implications on our ability to faithfully explain the behaviour of the underlying black-box model (arguably our principal goal). By essentially decoupling the task of learning plausible representations of the data from the model itself, we open ourselves up to vulnerabilities. Using a separate generative model to learn <span class="math inline">\(\mathcal{X}\)</span>, for example, has very serious implications for the generated counterfactuals. <a href="#fig-latent">Figure <span class="quarto-unresolved-ref">fig-latent</span></a> compares the results of applying REVISE <span class="citation" data-cites="joshi2019realistic">(<a href="#ref-joshi2019realistic" role="doc-biblioref">Joshi et al. 2019</a>)</span> to MNIST data using two different Variational Auto-Encoders: while the counterfactual generated using an expressive (strong) VAE is compelling, the result relying on a less expressive (weak) VAE is not even valid. In this latter case, the decoder step of the VAE fails to yield values in <span class="math inline">\(\mathcal{X}\)</span> and hence the counterfactual search in the learned latent space is doomed.</p>
-<div id="fig-latent" class="quarto-figure quarto-figure-center">
-<figure>
-<p><img src="www/mnist_9to4_latent.png" class="img-fluid" /></p>
-<p><figcaption>Figure 1.2: Counterfactual explanations for MNIST using a Latent Space generator: turning a nine (9) into a four (4).</figcaption></p>
-</figure>
-</div>
-<blockquote>
-<p>Here it would be nice to have another example where we poison the data going into the generative model to hide biases present in the data (e.g. Boston housing).</p>
-</blockquote>
-<ul>
-<li>Latent can be manipulated:
-<ul>
-<li>train biased model</li>
-<li>train VAE with biased variable removed/attacked (use Boston housing dataset)</li>
-<li>hypothesis: will generate bias-free explanations</li>
-</ul></li>
-</ul>
-</section>
-<section id="sec-fidelity" class="level3" data-number="1.1.2">
-<h3 data-number="1.1.2"><span class="header-section-number">1.1.2</span> From Plausible to High-Fidelity Counterfactuals</h3>
-<p>In light of the findings, we propose to generally avoid using surrogate models to learn <span class="math inline">\(\mathcal{X}\)</span> in the context of Counterfactual Explanations.</p>
-<div id="prp-surrogate" class="theorem proposition">
-<p><span class="theorem-title"><strong>Proposition 1.1 (Avoid Surrogates) </strong></span>Since we are in the business of explaining a black-box model, the task of learning realistic representations of the data should not be reallocated from the model itself to some surrogate model.</p>
-</div>
-<p>In cases where the use of surrogate models cannot be avoided, we propose to weigh the plausibility of counterfactuals against their fidelity to the black-box model. In the context of Explainable AI, fidelity is defined as describing how an explanation approximates the prediction of the black-box model <span class="citation" data-cites="molnar2020interpretable">(<a href="#ref-molnar2020interpretable" role="doc-biblioref">Molnar 2020</a>)</span>. Fidelity has become the default metric for evaluating Local Model-Agnostic Models, since they often involve local surrogate models whose predictions need not always match those of the black-box model.</p>
-<p>In the case of Counterfactual Explanations, the concept of fidelity has so far been ignored. This is not altogether surprising, since by construction and design, Counterfactual Explanations work with the predictions of the black-box model directly: as stated above, a counterfactual <span class="math inline">\(x^{\prime}\)</span> is considered valid if and only if <span class="math inline">\(f(x^{\prime})=t\)</span>, where <span class="math inline">\(t\)</span> denote some target outcome.</p>
-<p>Does fidelity even make sense in the context of CE, and if so, how can we define it? In light of the examples in the previous section, we think it is urgent to introduce a notion of fidelity in this context, that relates to the distributional properties of the generated counterfactuals. In particular, we propose that a high-fidelity counterfactual <span class="math inline">\(x^{\prime}\)</span> complies with the class-conditional distribution <span class="math inline">\(\mathcal{X}_{\theta} = p_{\theta}(X|y)\)</span> where <span class="math inline">\(\theta\)</span> denote the black-box model parameters.</p>
-<div id="def-fidele" class="theorem definition">
-<p><span class="theorem-title"><strong>Definition 1.2 (High-Fidelity Counterfactuals) </strong></span>Let <span class="math inline">\(\mathcal{X}_{\theta}|y = p_{\theta}(X|y)\)</span> denote the class-conditional distribution of <span class="math inline">\(X\)</span> defined by <span class="math inline">\(\theta\)</span>. Then for <span class="math inline">\(x^{\prime}\)</span> to be considered a high-fidelity counterfactual, we need: <span class="math inline">\(\mathcal{X}_{\theta}|t \approxeq \mathcal{X}^{\prime}\)</span> where <span class="math inline">\(t\)</span> denotes the target outcome.</p>
-</div>
-<p>In order to assess the fidelity of counterfactuals, we propose the following two-step procedure:</p>
-<ol type="1">
-<li>Generate samples <span class="math inline">\(X_{\theta}|y\)</span> and <span class="math inline">\(X^{\prime}\)</span> from <span class="math inline">\(\mathcal{X}_{\theta}|t\)</span> and <span class="math inline">\(\mathcal{X}^{\prime}\)</span>, respectively.</li>
-<li>Compute the Maximum Mean Discrepancy (MMD) between <span class="math inline">\(X_{\theta}|y\)</span> and <span class="math inline">\(X^{\prime}\)</span>.</li>
-</ol>
-<p>If the computed value is different from zero, we can reject the null-hypothesis of fidelity.</p>
-<blockquote>
-<p>Two challenges here: 1) implementing the sampling procedure in <span class="citation" data-cites="grathwohl2020your">Grathwohl et al. (<a href="#ref-grathwohl2020your" role="doc-biblioref">2020</a>)</span>; 2) it is unclear if MMD is really the right way to measure this.</p>
-</blockquote>
-</section>
-</section>
-<section id="conformal-counterfactual-explanations" class="level2" data-number="1.2">
-<h2 data-number="1.2"><span class="header-section-number">1.2</span> Conformal Counterfactual Explanations</h2>
-<p>In <a href="#sec-fidelity"><span class="quarto-unresolved-ref">sec-fidelity</span></a>, we have advocated for avoiding surrogate models in the context of Counterfactual Explanations. In this section, we introduce an alternative way to generate high-fidelity Counterfactual Explanations. In particular, we propose Conformal Counterfactual Explanations (CCE), that is Counterfactual Explanations that minimize the predictive uncertainty of conformal models.</p>
-<section id="minimizing-predictive-uncertainty" class="level3" data-number="1.2.1">
-<h3 data-number="1.2.1"><span class="header-section-number">1.2.1</span> Minimizing Predictive Uncertainty</h3>
-<p><span class="citation" data-cites="schut2021generating">Schut et al. (<a href="#ref-schut2021generating" role="doc-biblioref">2021</a>)</span> demonstrated that the goal of generating realistic (plausible) counterfactuals can also be achieved by seeking counterfactuals that minimize the predictive uncertainty of the underlying black-box model. Similarly, <span class="citation" data-cites="antoran2020getting">Antorán et al. (<a href="#ref-antoran2020getting" role="doc-biblioref">2020</a>)</span> …</p>
-<ul>
-<li>Problem: restricted to Bayesian models.</li>
-<li>Solution: post-hoc predictive uncertainty quantification. In particular, Conformal Prediction.</li>
-</ul>
-</section>
-<section id="background-on-conformal-prediction" class="level3" data-number="1.2.2">
-<h3 data-number="1.2.2"><span class="header-section-number">1.2.2</span> Background on Conformal Prediction</h3>
-<ul>
-<li>Distribution-free, model-agnostic and scalable approach to predictive uncertainty quantification.</li>
-<li>Conformal prediction is instance-based. So is CE.</li>
-<li>Take any fitted model and turn it into a conformal model using calibration data.</li>
-<li>Our approach, therefore, relaxes the restriction on the family of black-box models, at the cost of relying on a subset of the data. Arguably, data is often abundant and in most applications practitioners tend to hold out a test data set anyway.</li>
-</ul>
-<blockquote>
-<p>Does the coverage guarantee carry over to counterfactuals?</p>
-</blockquote>
-</section>
-<section id="generating-conformal-counterfactuals" class="level3" data-number="1.2.3">
-<h3 data-number="1.2.3"><span class="header-section-number">1.2.3</span> Generating Conformal Counterfactuals</h3>
-<p>While Conformal Prediction has recently grown in popularity, it does introduce a challenge in the context of classification: the predictions of Conformal Classifiers are set-valued and therefore difficult to work with, since they are, for example, non-differentiable. Fortunately, <span class="citation" data-cites="stutz2022learning">Stutz et al. (<a href="#ref-stutz2022learning" role="doc-biblioref">2022</a>)</span> introduced carefully designed differentiable loss functions that make it possible to evaluate the performance of conformal predictions in training. We can leverage these recent advances in the context of gradient-based counterfactual search …</p>
-<blockquote>
-<p>Challenge: still need to implement these loss functions.</p>
-</blockquote>
-</section>
-</section>
-<section id="experiments" class="level2" data-number="1.3">
-<h2 data-number="1.3"><span class="header-section-number">1.3</span> Experiments</h2>
-<section id="research-questions" class="level3" data-number="1.3.1">
-<h3 data-number="1.3.1"><span class="header-section-number">1.3.1</span> Research Questions</h3>
-<ul>
-<li><p>Is CP alone enough to ensure realistic counterfactuals?</p></li>
-<li><p>Do counterfactuals improve further as the models get better?</p></li>
-<li><p>Do counterfactuals get more realistic as coverage</p></li>
-<li><p>What happens as we vary coverage and setsize?</p></li>
-<li><p>What happens as we improve the model robustness?</p></li>
-<li><p>What happens as we improve the model’s ability to incorporate predictive uncertainty (deep ensemble, laplace)?</p></li>
-<li><p>What happens if we combine with DiCE, ClaPROAR, Gravitational?</p></li>
-<li><p>What about CE robustness to endogenous shifts <span class="citation" data-cites="altmeyer2023endogenous">(<a href="#ref-altmeyer2023endogenous" role="doc-biblioref">Altmeyer et al. 2023</a>)</span>?</p></li>
-<li><p>Benchmarking:</p>
-<ul>
-<li>add PROBE <span class="citation" data-cites="pawelczyk2022probabilistically">(<a href="#ref-pawelczyk2022probabilistically" role="doc-biblioref">Pawelczyk et al. 2022</a>)</span> into the mix.</li>
-<li>compare travel costs to domain shits.</li>
-</ul></li>
-</ul>
-<blockquote>
-<p>Nice to have: What about using Laplace Approximation, then Conformal Prediction? What about using Conformalised Laplace?</p>
-</blockquote>
-</section>
-</section>
-<section id="references" class="level2" data-number="1.4">
-<h2 data-number="1.4"><span class="header-section-number">1.4</span> References</h2>
-<div id="quarto-navigation-envelope" class="hidden">
-<p><span class="hidden" data-render-id="quarto-int-sidebar-title">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-int-navbar-title">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-int-next"><span class="chapter-number">2</span>Â <span class="chapter-title"><code>ConformalGenerator</code></span></span> <span class="hidden" data-render-id="quarto-int-prev">Preface</span> <span class="hidden" data-render-id="quarto-int-sidebar:/index.html">Preface</span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/proposal.html"><span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/intro.html"><span class="chapter-number">2</span>Â <span class="chapter-title"><code>ConformalGenerator</code></span></span> <span class="hidden" data-render-id="quarto-int-sidebar:/notebooks/references.html">References</span></p>
-</div>
-<div id="quarto-meta-markdown" class="hidden">
-<p><span class="hidden" data-render-id="quarto-metatitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-twittercardtitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-ogcardtitle">Conformal Counterfactual Explanations - <span class="chapter-number">1</span>Â <span class="chapter-title">High-Fidelity Counterfactual Explanations through Conformal Prediction</span></span> <span class="hidden" data-render-id="quarto-metasitename">Conformal Counterfactual Explanations</span> <span class="hidden" data-render-id="quarto-twittercarddesc">We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</span> <span class="hidden" data-render-id="quarto-ogcardddesc">We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.</span></p>
-</div>
-<div id="refs" class="references csl-bib-body hanging-indent" role="list">
-<div id="ref-altmeyer2023endogenous" class="csl-entry" role="listitem">
-Altmeyer, Patrick, Giovan Angela, Aleksander Buszydlik, Karol Dobiczek, Arie van Deursen, and Cynthia Liem. 2023. <span>“Endogenous <span>Macrodynamics</span> in <span>Algorithmic</span> <span>Recourse</span>.â€</span> In <em>First <span>IEEE</span> <span>Conference</span> on <span>Secure</span> and <span>Trustworthy</span> <span>Machine</span> <span>Learning</span></em>.
-</div>
-<div id="ref-antoran2020getting" class="csl-entry" role="listitem">
-Antorán, Javier, Umang Bhatt, Tameem Adel, Adrian Weller, and José Miguel Hernández-Lobato. 2020. <span>“Getting a Clue: <span>A</span> Method for Explaining Uncertainty Estimates.â€</span> <a href="https://arxiv.org/abs/2006.06848">https://arxiv.org/abs/2006.06848</a>.
-</div>
-<div id="ref-grathwohl2020your" class="csl-entry" role="listitem">
-Grathwohl, Will, Kuan-Chieh Wang, Joern-Henrik Jacobsen, David Duvenaud, Mohammad Norouzi, and Kevin Swersky. 2020. <span>“Your Classifier Is Secretly an Energy Based Model and You Should Treat It Like One.â€</span> In. <a href="https://openreview.net/forum?id=Hkxzx0NtDB">https://openreview.net/forum?id=Hkxzx0NtDB</a>.
-</div>
-<div id="ref-joshi2019realistic" class="csl-entry" role="listitem">
-Joshi, Shalmali, Oluwasanmi Koyejo, Warut Vijitbenjaronk, Been Kim, and Joydeep Ghosh. 2019. <span>“Towards Realistic Individual Recourse and Actionable Explanations in Black-Box Decision Making Systems.â€</span> <a href="https://arxiv.org/abs/1907.09615">https://arxiv.org/abs/1907.09615</a>.
-</div>
-<div id="ref-karimi2021algorithmic" class="csl-entry" role="listitem">
-Karimi, Amir-Hossein, Bernhard Schölkopf, and Isabel Valera. 2021. <span>“Algorithmic Recourse: From Counterfactual Explanations to Interventions.â€</span> In <em>Proceedings of the 2021 <span>ACM Conference</span> on <span>Fairness</span>, <span>Accountability</span>, and <span>Transparency</span></em>, 353–62.
-</div>
-<div id="ref-molnar2020interpretable" class="csl-entry" role="listitem">
-Molnar, Christoph. 2020. <em>Interpretable Machine Learning</em>. <span>Lulu. com</span>.
-</div>
-<div id="ref-pawelczyk2022probabilistically" class="csl-entry" role="listitem">
-Pawelczyk, Martin, Teresa Datta, Johannes van-den-Heuvel, Gjergji Kasneci, and Himabindu Lakkaraju. 2022. <span>“Probabilistically <span>Robust</span> <span>Recourse</span>: <span>Navigating</span> the <span>Trade</span>-Offs Between <span>Costs</span> and <span>Robustness</span> in <span>Algorithmic</span> <span>Recourse</span>.â€</span> <em>arXiv Preprint arXiv:2203.06768</em>.
-</div>
-<div id="ref-poyiadzi2020face" class="csl-entry" role="listitem">
-Poyiadzi, Rafael, Kacper Sokol, Raul Santos-Rodriguez, Tijl De Bie, and Peter Flach. 2020. <span>“<span>FACE</span>: <span>Feasible</span> and Actionable Counterfactual Explanations.â€</span> In <em>Proceedings of the <span>AAAI</span>/<span>ACM Conference</span> on <span>AI</span>, <span>Ethics</span>, and <span>Society</span></em>, 344–50.
-</div>
-<div id="ref-schut2021generating" class="csl-entry" role="listitem">
-Schut, Lisa, Oscar Key, Rory Mc Grath, Luca Costabello, Bogdan Sacaleanu, Yarin Gal, et al. 2021. <span>“Generating <span>Interpretable Counterfactual Explanations By Implicit Minimisation</span> of <span>Epistemic</span> and <span>Aleatoric Uncertainties</span>.â€</span> In <em>International <span>Conference</span> on <span>Artificial Intelligence</span> and <span>Statistics</span></em>, 1756–64. <span>PMLR</span>.
-</div>
-<div id="ref-stutz2022learning" class="csl-entry" role="listitem">
-Stutz, David, Krishnamurthy Dj Dvijotham, Ali Taylan Cemgil, and Arnaud Doucet. 2022. <span>“Learning <span>Optimal</span> <span>Conformal</span> <span>Classifiers</span>.â€</span> In. <a href="https://openreview.net/forum?id=t8O-4LKFVx">https://openreview.net/forum?id=t8O-4LKFVx</a>.
-</div>
-<div id="ref-wachter2017counterfactual" class="csl-entry" role="listitem">
-Wachter, Sandra, Brent Mittelstadt, and Chris Russell. 2017. <span>“Counterfactual Explanations Without Opening the Black Box: <span>Automated</span> Decisions and the <span>GDPR</span>.â€</span> <em>Harv. JL & Tech.</em> 31: 841.
-</div>
-</div>
-</section>
-
-</main> <!-- /main -->
-<script id = "quarto-html-after-body" type="application/javascript">
-window.document.addEventListener("DOMContentLoaded", function (event) {
- const toggleBodyColorMode = (bsSheetEl) => {
- const mode = bsSheetEl.getAttribute("data-mode");
- const bodyEl = window.document.querySelector("body");
- if (mode === "dark") {
- bodyEl.classList.add("quarto-dark");
- bodyEl.classList.remove("quarto-light");
- } else {
- bodyEl.classList.add("quarto-light");
- bodyEl.classList.remove("quarto-dark");
- }
- }
- const toggleBodyColorPrimary = () => {
- const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
- if (bsSheetEl) {
- toggleBodyColorMode(bsSheetEl);
- }
- }
- toggleBodyColorPrimary();
- const icon = "";
- const anchorJS = new window.AnchorJS();
- anchorJS.options = {
- placement: 'right',
- icon: icon
- };
- anchorJS.add('.anchored');
- const clipboard = new window.ClipboardJS('.code-copy-button', {
- target: function(trigger) {
- return trigger.previousElementSibling;
- }
- });
- clipboard.on('success', function(e) {
- // button target
- const button = e.trigger;
- // don't keep focus
- button.blur();
- // flash "checked"
- button.classList.add('code-copy-button-checked');
- var currentTitle = button.getAttribute("title");
- button.setAttribute("title", "Copied!");
- let tooltip;
- if (window.bootstrap) {
- button.setAttribute("data-bs-toggle", "tooltip");
- button.setAttribute("data-bs-placement", "left");
- button.setAttribute("data-bs-title", "Copied!");
- tooltip = new bootstrap.Tooltip(button,
- { trigger: "manual",
- customClass: "code-copy-button-tooltip",
- offset: [0, -8]});
- tooltip.show();
- }
- setTimeout(function() {
- if (tooltip) {
- tooltip.hide();
- button.removeAttribute("data-bs-title");
- button.removeAttribute("data-bs-toggle");
- button.removeAttribute("data-bs-placement");
- }
- button.setAttribute("title", currentTitle);
- button.classList.remove('code-copy-button-checked');
- }, 1000);
- // clear code selection
- e.clearSelection();
- });
- function tippyHover(el, contentFn) {
- const config = {
- allowHTML: true,
- content: contentFn,
- maxWidth: 500,
- delay: 100,
- arrow: false,
- appendTo: function(el) {
- return el.parentElement;
- },
- interactive: true,
- interactiveBorder: 10,
- theme: 'quarto',
- placement: 'bottom-start'
- };
- window.tippy(el, config);
- }
- const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
- for (var i=0; i<noterefs.length; i++) {
- const ref = noterefs[i];
- tippyHover(ref, function() {
- // use id or data attribute instead here
- let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
- try { href = new URL(href).hash; } catch {}
- const id = href.replace(/^#\/?/, "");
- const note = window.document.getElementById(id);
- return note.innerHTML;
- });
- }
- let selectedAnnoteEl;
- const selectorForAnnotation = ( cell, annotation) => {
- let cellAttr = 'data-code-cell="' + cell + '"';
- let lineAttr = 'data-code-annotation="' + annotation + '"';
- const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
- return selector;
- }
- const selectCodeLines = (annoteEl) => {
- const doc = window.document;
- const targetCell = annoteEl.getAttribute("data-target-cell");
- const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
- const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
- const lines = annoteSpan.getAttribute("data-code-lines").split(",");
- const lineIds = lines.map((line) => {
- return targetCell + "-" + line;
- })
- let top = null;
- let height = null;
- let parent = null;
- if (lineIds.length > 0) {
- //compute the position of the single el (top and bottom and make a div)
- const el = window.document.getElementById(lineIds[0]);
- top = el.offsetTop;
- height = el.offsetHeight;
- parent = el.parentElement.parentElement;
- if (lineIds.length > 1) {
- const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
- const bottom = lastEl.offsetTop + lastEl.offsetHeight;
- height = bottom - top;
- }
- if (top !== null && height !== null && parent !== null) {
- // cook up a div (if necessary) and position it
- let div = window.document.getElementById("code-annotation-line-highlight");
- if (div === null) {
- div = window.document.createElement("div");
- div.setAttribute("id", "code-annotation-line-highlight");
- div.style.position = 'absolute';
- parent.appendChild(div);
- }
- div.style.top = top - 2 + "px";
- div.style.height = height + 4 + "px";
- let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
- if (gutterDiv === null) {
- gutterDiv = window.document.createElement("div");
- gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
- gutterDiv.style.position = 'absolute';
- const codeCell = window.document.getElementById(targetCell);
- const gutter = codeCell.querySelector('.code-annotation-gutter');
- gutter.appendChild(gutterDiv);
- }
- gutterDiv.style.top = top - 2 + "px";
- gutterDiv.style.height = height + 4 + "px";
- }
- selectedAnnoteEl = annoteEl;
- }
- };
- const unselectCodeLines = () => {
- const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
- elementsIds.forEach((elId) => {
- const div = window.document.getElementById(elId);
- if (div) {
- div.remove();
- }
- });
- selectedAnnoteEl = undefined;
- };
- // Attach click handler to the DT
- const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
- for (const annoteDlNode of annoteDls) {
- annoteDlNode.addEventListener('click', (event) => {
- const clickedEl = event.target;
- if (clickedEl !== selectedAnnoteEl) {
- unselectCodeLines();
- const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
- if (activeEl) {
- activeEl.classList.remove('code-annotation-active');
- }
- selectCodeLines(clickedEl);
- clickedEl.classList.add('code-annotation-active');
- } else {
- // Unselect the line
- unselectCodeLines();
- clickedEl.classList.remove('code-annotation-active');
- }
- });
- }
- const findCites = (el) => {
- const parentEl = el.parentElement;
- if (parentEl) {
- const cites = parentEl.dataset.cites;
- if (cites) {
- return {
- el,
- cites: cites.split(' ')
- };
- } else {
- return findCites(el.parentElement)
- }
- } else {
- return undefined;
- }
- };
- var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
- for (var i=0; i<bibliorefs.length; i++) {
- const ref = bibliorefs[i];
- const citeInfo = findCites(ref);
- if (citeInfo) {
- tippyHover(citeInfo.el, function() {
- var popup = window.document.createElement('div');
- citeInfo.cites.forEach(function(cite) {
- var citeDiv = window.document.createElement('div');
- citeDiv.classList.add('hanging-indent');
- citeDiv.classList.add('csl-entry');
- var biblioDiv = window.document.getElementById('ref-' + cite);
- if (biblioDiv) {
- citeDiv.innerHTML = biblioDiv.innerHTML;
- }
- popup.appendChild(citeDiv);
- });
- return popup.innerHTML;
- });
- }
- }
-});
-</script>
-<nav class="page-navigation">
- <div class="nav-page nav-page-previous">
- <a href="/index.html" class="pagination-link">
- <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text">Preface</span>
- </a>
- </div>
- <div class="nav-page nav-page-next">
- <a href="/notebooks/intro.html" class="pagination-link">
- <span class="nav-page-text"><span class='chapter-number'>2</span>Â <span class='chapter-title'>`ConformalGenerator`</span></span> <i class="bi bi-arrow-right-short"></i>
- </a>
- </div>
-</nav>
-</div> <!-- /content -->
-
-</body>
-
-</html>
\ No newline at end of file
diff --git a/notebooks/proposal.ipynb b/notebooks/proposal.ipynb
deleted file mode 100644
index bf087486e317151e52c202913565ac26aef95b5d..0000000000000000000000000000000000000000
--- a/notebooks/proposal.ipynb
+++ /dev/null
@@ -1,218 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "raw",
- "metadata": {},
- "source": [
- "---\n",
- "title: High-Fidelity Counterfactual Explanations through Conformal Prediction\n",
- "subtitle: Research Proposal\n",
- "abstract: |\n",
- " We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.\n",
- "---"
- ],
- "id": "0a271b6a"
- },
- {
- "cell_type": "code",
- "metadata": {},
- "source": [
- "include(\"notebooks/setup.jl\")\n",
- "eval(setup_notebooks)"
- ],
- "id": "3f81501f",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Motivation\n",
- "\n",
- "Counterfactual Explanations are a powerful, flexible and intuitive way to not only explain black-box models but also enable affected individuals to challenge them through the means of Algorithmic Recourse. \n",
- "\n",
- "### Counterfactual Explanations or Adversarial Examples?\n",
- "\n",
- "Most state-of-the-art approaches to generating Counterfactual Explanations (CE) rely on gradient descent in the feature space. The key idea is to perturb inputs $x\\in\\mathcal{X}$ into a black-box model $f: \\mathcal{X} \\mapsto \\mathcal{Y}$ in order to change the model output $f(x)$ to some pre-specified target value $t\\in\\mathcal{Y}$. Formally, this boils down to defining some loss function $\\ell(f(x),t)$ and taking gradient steps in the minimizing direction. The so-generated counterfactuals are considered valid as soon as the predicted label matches the target label. A stripped-down counterfactual explanation is therefore little different from an adversarial example. In @fig-adv, for example, generic counterfactual search as in @wachter2017counterfactual has been applied to MNIST data.\n"
- ],
- "id": "a244bee7"
- },
- {
- "cell_type": "code",
- "metadata": {},
- "source": [
- "# Data:\n",
- "counterfactual_data = load_mnist()\n",
- "X, y = CounterfactualExplanations.DataPreprocessing.unpack_data(counterfactual_data)\n",
- "input_dim, n_obs = size(counterfactual_data.X)\n",
- "M = load_mnist_mlp()\n",
- "# Target:\n",
- "factual_label = 8\n",
- "x = reshape(X[:,rand(findall(predict_label(M, counterfactual_data).==factual_label))],input_dim,1)\n",
- "target = 3\n",
- "factual = predict_label(M, counterfactual_data, x)[1]\n",
- "# Search:\n",
- "n_ce = 3\n",
- "generator = GenericGenerator()\n",
- "ces = generate_counterfactual(x, target, counterfactual_data, M, generator; num_counterfactuals=n_ce)"
- ],
- "id": "687cba09",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "metadata": {},
- "source": [
- "image_size = 200\n",
- "p1 = plot(\n",
- " convert2image(MNIST, reshape(x,28,28)),\n",
- " axis=nothing, \n",
- " size=(image_size, image_size),\n",
- " title=\"Factual\"\n",
- ")\n",
- "plts = [p1]\n",
- "\n",
- "counterfactuals = CounterfactualExplanations.counterfactual(ces)\n",
- "phat = target_probs(ces)\n",
- "for x in zip(eachslice(counterfactuals; dims=3), eachslice(phat; dims=3))\n",
- " ce, _phat = (x[1],x[2])\n",
- " _title = \"p(y=$(target)|x′)=$(round(_phat[1]; digits=3))\"\n",
- " plt = plot(\n",
- " convert2image(MNIST, reshape(ce,28,28)),\n",
- " axis=nothing, \n",
- " size=(image_size, image_size),\n",
- " title=_title\n",
- " )\n",
- " plts = [plts..., plt]\n",
- "end\n",
- "plt = plot(plts...; size=(image_size * (n_ce + 1),image_size), layout=(1,(n_ce + 1)))\n",
- "savefig(plt, joinpath(www_path, \"you_may_not_like_it.png\"))"
- ],
- "id": "fda7c987",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "{#fig-adv}\n",
- "\n",
- "The crucial difference between adversarial examples and counterfactuals is one of intent. While adversarial examples are typically intended to go unnoticed, counterfactuals in the context of Explainable AI are generally sought to be \"plausible\", \"realistic\" or \"feasible\". To fulfil this latter goal, researchers have come up with a myriad of ways. @joshi2019realistic were among the first to suggest that instead of searching counterfactuals in the feature space, we can instead traverse a latent embedding learned by a surrogate generative model. Similarly, @poyiadzi2020face use density ... Finally, @karimi2021algorithmic argues that counterfactuals should comply with the causal model that generates them [CHECK IF WE CAN PHASE THIS LIKE THIS]. Other related approaches include ... All of these different approaches have a common goal: they aim to ensure that the generated counterfactuals comply with the (learned) data-generating process (DGB). \n",
- "\n",
- "::: {#def-plausible}\n",
- "\n",
- "## Plausible Counterfactuals\n",
- "\n",
- "Formally, if $x \\sim \\mathcal{X}$ and for the corresponding counterfactual we have $x^{\\prime}\\sim\\mathcal{X}^{\\prime}$, then for $x^{\\prime}$ to be considered a plausible counterfactual, we need: $\\mathcal{X} \\approxeq \\mathcal{X}^{\\prime}$.\n",
- "\n",
- ":::\n",
- "\n",
- "In the context of Algorithmic Recourse, it makes sense to strive for plausible counterfactuals, since anything else would essentially require individuals to move to out-of-distribution states. But it is worth noting that our ambition to meet this goal, may have implications on our ability to faithfully explain the behaviour of the underlying black-box model (arguably our principal goal). By essentially decoupling the task of learning plausible representations of the data from the model itself, we open ourselves up to vulnerabilities. Using a separate generative model to learn $\\mathcal{X}$, for example, has very serious implications for the generated counterfactuals. @fig-latent compares the results of applying REVISE [@joshi2019realistic] to MNIST data using two different Variational Auto-Encoders: while the counterfactual generated using an expressive (strong) VAE is compelling, the result relying on a less expressive (weak) VAE is not even valid. In this latter case, the decoder step of the VAE fails to yield values in $\\mathcal{X}$ and hence the counterfactual search in the learned latent space is doomed. \n",
- "\n",
- "{#fig-latent}\n",
- "\n",
- "> Here it would be nice to have another example where we poison the data going into the generative model to hide biases present in the data (e.g. Boston housing).\n",
- "\n",
- "- Latent can be manipulated: \n",
- " - train biased model\n",
- " - train VAE with biased variable removed/attacked (use Boston housing dataset)\n",
- " - hypothesis: will generate bias-free explanations\n",
- "\n",
- "### From Plausible to High-Fidelity Counterfactuals {#sec-fidelity}\n",
- "\n",
- "In light of the findings, we propose to generally avoid using surrogate models to learn $\\mathcal{X}$ in the context of Counterfactual Explanations.\n",
- "\n",
- "::: {#prp-surrogate}\n",
- "\n",
- "## Avoid Surrogates\n",
- "\n",
- "Since we are in the business of explaining a black-box model, the task of learning realistic representations of the data should not be reallocated from the model itself to some surrogate model.\n",
- "\n",
- ":::\n",
- "\n",
- "In cases where the use of surrogate models cannot be avoided, we propose to weigh the plausibility of counterfactuals against their fidelity to the black-box model. In the context of Explainable AI, fidelity is defined as describing how an explanation approximates the prediction of the black-box model [@molnar2020interpretable]. Fidelity has become the default metric for evaluating Local Model-Agnostic Models, since they often involve local surrogate models whose predictions need not always match those of the black-box model. \n",
- "\n",
- "In the case of Counterfactual Explanations, the concept of fidelity has so far been ignored. This is not altogether surprising, since by construction and design, Counterfactual Explanations work with the predictions of the black-box model directly: as stated above, a counterfactual $x^{\\prime}$ is considered valid if and only if $f(x^{\\prime})=t$, where $t$ denote some target outcome. \n",
- "\n",
- "Does fidelity even make sense in the context of CE, and if so, how can we define it? In light of the examples in the previous section, we think it is urgent to introduce a notion of fidelity in this context, that relates to the distributional properties of the generated counterfactuals. In particular, we propose that a high-fidelity counterfactual $x^{\\prime}$ complies with the class-conditional distribution $\\mathcal{X}_{\\theta} = p_{\\theta}(X|y)$ where $\\theta$ denote the black-box model parameters. \n",
- "\n",
- "::: {#def-fidele}\n",
- "\n",
- "## High-Fidelity Counterfactuals\n",
- "\n",
- "Let $\\mathcal{X}_{\\theta}|y = p_{\\theta}(X|y)$ denote the class-conditional distribution of $X$ defined by $\\theta$. Then for $x^{\\prime}$ to be considered a high-fidelity counterfactual, we need: $\\mathcal{X}_{\\theta}|t \\approxeq \\mathcal{X}^{\\prime}$ where $t$ denotes the target outcome.\n",
- "\n",
- ":::\n",
- "\n",
- "In order to assess the fidelity of counterfactuals, we propose the following two-step procedure:\n",
- "\n",
- "1) Generate samples $X_{\\theta}|y$ and $X^{\\prime}$ from $\\mathcal{X}_{\\theta}|t$ and $\\mathcal{X}^{\\prime}$, respectively.\n",
- "2) Compute the Maximum Mean Discrepancy (MMD) between $X_{\\theta}|y$ and $X^{\\prime}$. \n",
- "\n",
- "If the computed value is different from zero, we can reject the null-hypothesis of fidelity.\n",
- "\n",
- "> Two challenges here: 1) implementing the sampling procedure in @grathwohl2020your; 2) it is unclear if MMD is really the right way to measure this. \n",
- "\n",
- "## Conformal Counterfactual Explanations\n",
- "\n",
- "In @sec-fidelity, we have advocated for avoiding surrogate models in the context of Counterfactual Explanations. In this section, we introduce an alternative way to generate high-fidelity Counterfactual Explanations. In particular, we propose Conformal Counterfactual Explanations (CCE), that is Counterfactual Explanations that minimize the predictive uncertainty of conformal models. \n",
- "\n",
- "### Minimizing Predictive Uncertainty\n",
- "\n",
- "@schut2021generating demonstrated that the goal of generating realistic (plausible) counterfactuals can also be achieved by seeking counterfactuals that minimize the predictive uncertainty of the underlying black-box model. Similarly, @antoran2020getting ...\n",
- "\n",
- "- Problem: restricted to Bayesian models.\n",
- "- Solution: post-hoc predictive uncertainty quantification. In particular, Conformal Prediction. \n",
- "\n",
- "### Background on Conformal Prediction\n",
- "\n",
- "- Distribution-free, model-agnostic and scalable approach to predictive uncertainty quantification.\n",
- "- Conformal prediction is instance-based. So is CE. \n",
- "- Take any fitted model and turn it into a conformal model using calibration data.\n",
- "- Our approach, therefore, relaxes the restriction on the family of black-box models, at the cost of relying on a subset of the data. Arguably, data is often abundant and in most applications practitioners tend to hold out a test data set anyway. \n",
- "\n",
- "> Does the coverage guarantee carry over to counterfactuals?\n",
- "\n",
- "### Generating Conformal Counterfactuals\n",
- "\n",
- "While Conformal Prediction has recently grown in popularity, it does introduce a challenge in the context of classification: the predictions of Conformal Classifiers are set-valued and therefore difficult to work with, since they are, for example, non-differentiable. Fortunately, @stutz2022learning introduced carefully designed differentiable loss functions that make it possible to evaluate the performance of conformal predictions in training. We can leverage these recent advances in the context of gradient-based counterfactual search ...\n",
- "\n",
- "> Challenge: still need to implement these loss functions. \n",
- "\n",
- "## Experiments\n",
- "\n",
- "### Research Questions\n",
- "\n",
- "- Is CP alone enough to ensure realistic counterfactuals?\n",
- "- Do counterfactuals improve further as the models get better?\n",
- "- Do counterfactuals get more realistic as coverage\n",
- "- What happens as we vary coverage and setsize?\n",
- "- What happens as we improve the model robustness?\n",
- "- What happens as we improve the model's ability to incorporate predictive uncertainty (deep ensemble, laplace)?\n",
- "- What happens if we combine with DiCE, ClaPROAR, Gravitational?\n",
- "- What about CE robustness to endogenous shifts [@altmeyer2023endogenous]?\n",
- "\n",
- "- Benchmarking:\n",
- " - add PROBE [@pawelczyk2022probabilistically] into the mix.\n",
- " - compare travel costs to domain shits.\n",
- "\n",
- "> Nice to have: What about using Laplace Approximation, then Conformal Prediction? What about using Conformalised Laplace? \n",
- "\n",
- "## References\n"
- ],
- "id": "cb684712"
- }
- ],
- "metadata": {
- "kernelspec": {
- "name": "julia-(4-threads)-1.8",
- "language": "julia",
- "display_name": "Julia (4 threads) 1.8.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
\ No newline at end of file
diff --git a/notebooks/proposal_files/execute-results/html.json b/notebooks/proposal_files/execute-results/html.json
new file mode 100644
index 0000000000000000000000000000000000000000..9f577ad988fa3da68878485a4db4fa7e7d88bc5b
--- /dev/null
+++ b/notebooks/proposal_files/execute-results/html.json
@@ -0,0 +1,11 @@
+{
+ "hash": "b359927c37941b81cb7bf792790a86f8",
+ "result": {
+ "markdown": "---\ntitle: High-Fidelity Counterfactual Explanations through Conformal Prediction\nsubtitle: Research Proposal\nabstract: |\n We propose Conformal Counterfactual Explanations: an effortless and rigorous way to produce realistic and faithful Counterfactual Explanations using Conformal Prediction. To address the need for realistic counterfactuals, existing work has primarily relied on separate generative models to learn the data-generating process. While this is an effective way to produce plausible and model-agnostic counterfactual explanations, it not only introduces a significant engineering overhead but also reallocates the task of creating realistic model explanations from the model itself to the generative model. Recent work has shown that there is no need for any of this when working with probabilistic models that explicitly quantify their own uncertainty. Unfortunately, most models used in practice still do not fulfil that basic requirement, in which case we would like to have a way to quantify predictive uncertainty in a post-hoc fashion.\n---\n\n\n\n## Motivation\n\nCounterfactual Explanations are a powerful, flexible and intuitive way to not only explain black-box models but also enable affected individuals to challenge them through the means of Algorithmic Recourse. \n\n### Counterfactual Explanations or Adversarial Examples?\n\nMost state-of-the-art approaches to generating Counterfactual Explanations (CE) rely on gradient descent in the feature space. The key idea is to perturb inputs $x\\in\\mathcal{X}$ into a black-box model $f: \\mathcal{X} \\mapsto \\mathcal{Y}$ in order to change the model output $f(x)$ to some pre-specified target value $t\\in\\mathcal{Y}$. Formally, this boils down to defining some loss function $\\ell(f(x),t)$ and taking gradient steps in the minimizing direction. The so-generated counterfactuals are considered valid as soon as the predicted label matches the target label. A stripped-down counterfactual explanation is therefore little different from an adversarial example. In @fig-adv, for example, generic counterfactual search as in @wachter2017counterfactual has been applied to MNIST data.\n\n\n\n\n\n{#fig-adv}\n\nThe crucial difference between adversarial examples and counterfactuals is one of intent. While adversarial examples are typically intended to go unnoticed, counterfactuals in the context of Explainable AI are generally sought to be \"plausible\", \"realistic\" or \"feasible\". To fulfil this latter goal, researchers have come up with a myriad of ways. @joshi2019realistic were among the first to suggest that instead of searching counterfactuals in the feature space, we can instead traverse a latent embedding learned by a surrogate generative model. Similarly, @poyiadzi2020face use density ... Finally, @karimi2021algorithmic argues that counterfactuals should comply with the causal model that generates them [CHECK IF WE CAN PHASE THIS LIKE THIS]. Other related approaches include ... All of these different approaches have a common goal: they aim to ensure that the generated counterfactuals comply with the (learned) data-generating process (DGB). \n\n::: {#def-plausible}\n\n## Plausible Counterfactuals\n\nFormally, if $x \\sim \\mathcal{X}$ and for the corresponding counterfactual we have $x^{\\prime}\\sim\\mathcal{X}^{\\prime}$, then for $x^{\\prime}$ to be considered a plausible counterfactual, we need: $\\mathcal{X} \\approxeq \\mathcal{X}^{\\prime}$.\n\n:::\n\nIn the context of Algorithmic Recourse, it makes sense to strive for plausible counterfactuals, since anything else would essentially require individuals to move to out-of-distribution states. But it is worth noting that our ambition to meet this goal, may have implications on our ability to faithfully explain the behaviour of the underlying black-box model (arguably our principal goal). By essentially decoupling the task of learning plausible representations of the data from the model itself, we open ourselves up to vulnerabilities. Using a separate generative model to learn $\\mathcal{X}$, for example, has very serious implications for the generated counterfactuals. @fig-latent compares the results of applying REVISE [@joshi2019realistic] to MNIST data using two different Variational Auto-Encoders: while the counterfactual generated using an expressive (strong) VAE is compelling, the result relying on a less expressive (weak) VAE is not even valid. In this latter case, the decoder step of the VAE fails to yield values in $\\mathcal{X}$ and hence the counterfactual search in the learned latent space is doomed. \n\n{#fig-latent}\n\n> Here it would be nice to have another example where we poison the data going into the generative model to hide biases present in the data (e.g. Boston housing).\n\n- Latent can be manipulated: \n - train biased model\n - train VAE with biased variable removed/attacked (use Boston housing dataset)\n - hypothesis: will generate bias-free explanations\n\n### From Plausible to High-Fidelity Counterfactuals {#sec-fidelity}\n\nIn light of the findings, we propose to generally avoid using surrogate models to learn $\\mathcal{X}$ in the context of Counterfactual Explanations.\n\n::: {#prp-surrogate}\n\n## Avoid Surrogates\n\nSince we are in the business of explaining a black-box model, the task of learning realistic representations of the data should not be reallocated from the model itself to some surrogate model.\n\n:::\n\nIn cases where the use of surrogate models cannot be avoided, we propose to weigh the plausibility of counterfactuals against their fidelity to the black-box model. In the context of Explainable AI, fidelity is defined as describing how an explanation approximates the prediction of the black-box model [@molnar2020interpretable]. Fidelity has become the default metric for evaluating Local Model-Agnostic Models, since they often involve local surrogate models whose predictions need not always match those of the black-box model. \n\nIn the case of Counterfactual Explanations, the concept of fidelity has so far been ignored. This is not altogether surprising, since by construction and design, Counterfactual Explanations work with the predictions of the black-box model directly: as stated above, a counterfactual $x^{\\prime}$ is considered valid if and only if $f(x^{\\prime})=t$, where $t$ denote some target outcome. \n\nDoes fidelity even make sense in the context of CE, and if so, how can we define it? In light of the examples in the previous section, we think it is urgent to introduce a notion of fidelity in this context, that relates to the distributional properties of the generated counterfactuals. In particular, we propose that a high-fidelity counterfactual $x^{\\prime}$ complies with the class-conditional distribution $\\mathcal{X}_{\\theta} = p_{\\theta}(X|y)$ where $\\theta$ denote the black-box model parameters. \n\n::: {#def-fidele}\n\n## High-Fidelity Counterfactuals\n\nLet $\\mathcal{X}_{\\theta}|y = p_{\\theta}(X|y)$ denote the class-conditional distribution of $X$ defined by $\\theta$. Then for $x^{\\prime}$ to be considered a high-fidelity counterfactual, we need: $\\mathcal{X}_{\\theta}|t \\approxeq \\mathcal{X}^{\\prime}$ where $t$ denotes the target outcome.\n\n:::\n\nIn order to assess the fidelity of counterfactuals, we propose the following two-step procedure:\n\n1) Generate samples $X_{\\theta}|y$ and $X^{\\prime}$ from $\\mathcal{X}_{\\theta}|t$ and $\\mathcal{X}^{\\prime}$, respectively.\n2) Compute the Maximum Mean Discrepancy (MMD) between $X_{\\theta}|y$ and $X^{\\prime}$. \n\nIf the computed value is different from zero, we can reject the null-hypothesis of fidelity.\n\n> Two challenges here: 1) implementing the sampling procedure in @grathwohl2020your; 2) it is unclear if MMD is really the right way to measure this. \n\n## Conformal Counterfactual Explanations\n\nIn @sec-fidelity, we have advocated for avoiding surrogate models in the context of Counterfactual Explanations. In this section, we introduce an alternative way to generate high-fidelity Counterfactual Explanations. In particular, we propose Conformal Counterfactual Explanations (CCE), that is Counterfactual Explanations that minimize the predictive uncertainty of conformal models. \n\n### Minimizing Predictive Uncertainty\n\n@schut2021generating demonstrated that the goal of generating realistic (plausible) counterfactuals can also be achieved by seeking counterfactuals that minimize the predictive uncertainty of the underlying black-box model. Similarly, @antoran2020getting ...\n\n- Problem: restricted to Bayesian models.\n- Solution: post-hoc predictive uncertainty quantification. In particular, Conformal Prediction. \n\n### Background on Conformal Prediction\n\n- Distribution-free, model-agnostic and scalable approach to predictive uncertainty quantification.\n- Conformal prediction is instance-based. So is CE. \n- Take any fitted model and turn it into a conformal model using calibration data.\n- Our approach, therefore, relaxes the restriction on the family of black-box models, at the cost of relying on a subset of the data. Arguably, data is often abundant and in most applications practitioners tend to hold out a test data set anyway. \n\n> Does the coverage guarantee carry over to counterfactuals?\n\n### Generating Conformal Counterfactuals\n\nWhile Conformal Prediction has recently grown in popularity, it does introduce a challenge in the context of classification: the predictions of Conformal Classifiers are set-valued and therefore difficult to work with, since they are, for example, non-differentiable. Fortunately, @stutz2022learning introduced carefully designed differentiable loss functions that make it possible to evaluate the performance of conformal predictions in training. We can leverage these recent advances in the context of gradient-based counterfactual search ...\n\n> Challenge: still need to implement these loss functions. \n\n## Experiments\n\n### Research Questions\n\n- Is CP alone enough to ensure realistic counterfactuals?\n- Do counterfactuals improve further as the models get better?\n- Do counterfactuals get more realistic as coverage\n- What happens as we vary coverage and setsize?\n- What happens as we improve the model robustness?\n- What happens as we improve the model's ability to incorporate predictive uncertainty (deep ensemble, laplace)?\n- What happens if we combine with DiCE, ClaPROAR, Gravitational?\n- What about CE robustness to endogenous shifts [@altmeyer2023endogenous]?\n\n- Benchmarking:\n - add PROBE [@pawelczyk2022probabilistically] into the mix.\n - compare travel costs to domain shits.\n\n> Nice to have: What about using Laplace Approximation, then Conformal Prediction? What about using Conformalised Laplace? \n\n## References\n\n",
+ "supporting": [
+ "proposal_files/figure-html"
+ ],
+ "filters": [],
+ "includes": {}
+ }
+}
\ No newline at end of file
diff --git a/notebooks/www/you_may_not_like_it.png b/notebooks/www/you_may_not_like_it.png
index 78cc0733619723f5a0b54693a978ae2ce5123cbf..2e24f50d942c62ceed7c4e5990a0917eb807d9ba 100644
Binary files a/notebooks/www/you_may_not_like_it.png and b/notebooks/www/you_may_not_like_it.png differ
diff --git a/paper/neurips_2022.sty b/paper/neurips_2022.sty
new file mode 100644
index 0000000000000000000000000000000000000000..91568e27064c1f1f560262a8b4fe871084963447
--- /dev/null
+++ b/paper/neurips_2022.sty
@@ -0,0 +1,381 @@
+% partial rewrite of the LaTeX2e package for submissions to the
+% Conference on Neural Information Processing Systems (NeurIPS):
+%
+% - uses more LaTeX conventions
+% - line numbers at submission time replaced with aligned numbers from
+% lineno package
+% - \nipsfinalcopy replaced with [final] package option
+% - automatically loads times package for authors
+% - loads natbib automatically; this can be suppressed with the
+% [nonatbib] package option
+% - adds foot line to first page identifying the conference
+% - adds preprint option for submission to e.g. arXiv
+% - conference acronym modified
+%
+% Roman Garnett (garnett@wustl.edu) and the many authors of
+% nips15submit_e.sty, including MK and drstrip@sandia
+%
+% last revision: March 2022
+
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesPackage{neurips_2022}[2022/03/31 NeurIPS 2022 submission/camera-ready style file]
+
+% declare final option, which creates camera-ready copy
+\newif\if@neuripsfinal\@neuripsfinalfalse
+\DeclareOption{final}{
+ \@neuripsfinaltrue
+}
+
+% declare nonatbib option, which does not load natbib in case of
+% package clash (users can pass options to natbib via
+% \PassOptionsToPackage)
+\newif\if@natbib\@natbibtrue
+\DeclareOption{nonatbib}{
+ \@natbibfalse
+}
+
+% declare preprint option, which creates a preprint version ready for
+% upload to, e.g., arXiv
+\newif\if@preprint\@preprintfalse
+\DeclareOption{preprint}{
+ \@preprinttrue
+}
+
+\ProcessOptions\relax
+
+% determine whether this is an anonymized submission
+\newif\if@submission\@submissiontrue
+\if@neuripsfinal\@submissionfalse\fi
+\if@preprint\@submissionfalse\fi
+
+% fonts
+\renewcommand{\rmdefault}{ptm}
+\renewcommand{\sfdefault}{phv}
+
+% change this every year for notice string at bottom
+\newcommand{\@neuripsordinal}{36th}
+\newcommand{\@neuripsyear}{2022}
+\newcommand{\@neuripslocation}{New Orleans}
+
+% acknowledgments
+\usepackage{environ}
+\newcommand{\acksection}{\section*{Acknowledgments and Disclosure of Funding}}
+\NewEnviron{ack}{%
+ \acksection
+ \BODY
+}
+
+
+% load natbib unless told otherwise
+\if@natbib
+ \RequirePackage{natbib}
+\fi
+
+% set page geometry
+\usepackage[verbose=true,letterpaper]{geometry}
+\AtBeginDocument{
+ \newgeometry{
+ textheight=9in,
+ textwidth=5.5in,
+ top=1in,
+ headheight=12pt,
+ headsep=25pt,
+ footskip=30pt
+ }
+ \@ifpackageloaded{fullpage}
+ {\PackageWarning{neurips_2022}{fullpage package not allowed! Overwriting formatting.}}
+ {}
+}
+
+\widowpenalty=10000
+\clubpenalty=10000
+\flushbottom
+\sloppy
+
+
+% font sizes with reduced leading
+\renewcommand{\normalsize}{%
+ \@setfontsize\normalsize\@xpt\@xipt
+ \abovedisplayskip 7\p@ \@plus 2\p@ \@minus 5\p@
+ \abovedisplayshortskip \z@ \@plus 3\p@
+ \belowdisplayskip \abovedisplayskip
+ \belowdisplayshortskip 4\p@ \@plus 3\p@ \@minus 3\p@
+}
+\normalsize
+\renewcommand{\small}{%
+ \@setfontsize\small\@ixpt\@xpt
+ \abovedisplayskip 6\p@ \@plus 1.5\p@ \@minus 4\p@
+ \abovedisplayshortskip \z@ \@plus 2\p@
+ \belowdisplayskip \abovedisplayskip
+ \belowdisplayshortskip 3\p@ \@plus 2\p@ \@minus 2\p@
+}
+\renewcommand{\footnotesize}{\@setfontsize\footnotesize\@ixpt\@xpt}
+\renewcommand{\scriptsize}{\@setfontsize\scriptsize\@viipt\@viiipt}
+\renewcommand{\tiny}{\@setfontsize\tiny\@vipt\@viipt}
+\renewcommand{\large}{\@setfontsize\large\@xiipt{14}}
+\renewcommand{\Large}{\@setfontsize\Large\@xivpt{16}}
+\renewcommand{\LARGE}{\@setfontsize\LARGE\@xviipt{20}}
+\renewcommand{\huge}{\@setfontsize\huge\@xxpt{23}}
+\renewcommand{\Huge}{\@setfontsize\Huge\@xxvpt{28}}
+
+% sections with less space
+\providecommand{\section}{}
+\renewcommand{\section}{%
+ \@startsection{section}{1}{\z@}%
+ {-2.0ex \@plus -0.5ex \@minus -0.2ex}%
+ { 1.5ex \@plus 0.3ex \@minus 0.2ex}%
+ {\large\bf\raggedright}%
+}
+\providecommand{\subsection}{}
+\renewcommand{\subsection}{%
+ \@startsection{subsection}{2}{\z@}%
+ {-1.8ex \@plus -0.5ex \@minus -0.2ex}%
+ { 0.8ex \@plus 0.2ex}%
+ {\normalsize\bf\raggedright}%
+}
+\providecommand{\subsubsection}{}
+\renewcommand{\subsubsection}{%
+ \@startsection{subsubsection}{3}{\z@}%
+ {-1.5ex \@plus -0.5ex \@minus -0.2ex}%
+ { 0.5ex \@plus 0.2ex}%
+ {\normalsize\bf\raggedright}%
+}
+\providecommand{\paragraph}{}
+\renewcommand{\paragraph}{%
+ \@startsection{paragraph}{4}{\z@}%
+ {1.5ex \@plus 0.5ex \@minus 0.2ex}%
+ {-1em}%
+ {\normalsize\bf}%
+}
+\providecommand{\subparagraph}{}
+\renewcommand{\subparagraph}{%
+ \@startsection{subparagraph}{5}{\z@}%
+ {1.5ex \@plus 0.5ex \@minus 0.2ex}%
+ {-1em}%
+ {\normalsize\bf}%
+}
+\providecommand{\subsubsubsection}{}
+\renewcommand{\subsubsubsection}{%
+ \vskip5pt{\noindent\normalsize\rm\raggedright}%
+}
+
+% float placement
+\renewcommand{\topfraction }{0.85}
+\renewcommand{\bottomfraction }{0.4}
+\renewcommand{\textfraction }{0.1}
+\renewcommand{\floatpagefraction}{0.7}
+
+\newlength{\@neuripsabovecaptionskip}\setlength{\@neuripsabovecaptionskip}{7\p@}
+\newlength{\@neuripsbelowcaptionskip}\setlength{\@neuripsbelowcaptionskip}{\z@}
+
+\setlength{\abovecaptionskip}{\@neuripsabovecaptionskip}
+\setlength{\belowcaptionskip}{\@neuripsbelowcaptionskip}
+
+% swap above/belowcaptionskip lengths for tables
+\renewenvironment{table}
+ {\setlength{\abovecaptionskip}{\@neuripsbelowcaptionskip}%
+ \setlength{\belowcaptionskip}{\@neuripsabovecaptionskip}%
+ \@float{table}}
+ {\end@float}
+
+% footnote formatting
+\setlength{\footnotesep }{6.65\p@}
+\setlength{\skip\footins}{9\p@ \@plus 4\p@ \@minus 2\p@}
+\renewcommand{\footnoterule}{\kern-3\p@ \hrule width 12pc \kern 2.6\p@}
+\setcounter{footnote}{0}
+
+% paragraph formatting
+\setlength{\parindent}{\z@}
+\setlength{\parskip }{5.5\p@}
+
+% list formatting
+\setlength{\topsep }{4\p@ \@plus 1\p@ \@minus 2\p@}
+\setlength{\partopsep }{1\p@ \@plus 0.5\p@ \@minus 0.5\p@}
+\setlength{\itemsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@}
+\setlength{\parsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@}
+\setlength{\leftmargin }{3pc}
+\setlength{\leftmargini }{\leftmargin}
+\setlength{\leftmarginii }{2em}
+\setlength{\leftmarginiii}{1.5em}
+\setlength{\leftmarginiv }{1.0em}
+\setlength{\leftmarginv }{0.5em}
+\def\@listi {\leftmargin\leftmargini}
+\def\@listii {\leftmargin\leftmarginii
+ \labelwidth\leftmarginii
+ \advance\labelwidth-\labelsep
+ \topsep 2\p@ \@plus 1\p@ \@minus 0.5\p@
+ \parsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii
+ \advance\labelwidth-\labelsep
+ \topsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@
+ \parsep \z@
+ \partopsep 0.5\p@ \@plus 0\p@ \@minus 0.5\p@
+ \itemsep \topsep}
+\def\@listiv {\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv
+ \advance\labelwidth-\labelsep}
+\def\@listv {\leftmargin\leftmarginv
+ \labelwidth\leftmarginv
+ \advance\labelwidth-\labelsep}
+\def\@listvi {\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi
+ \advance\labelwidth-\labelsep}
+
+% create title
+\providecommand{\maketitle}{}
+\renewcommand{\maketitle}{%
+ \par
+ \begingroup
+ \renewcommand{\thefootnote}{\fnsymbol{footnote}}
+ % for perfect author name centering
+ \renewcommand{\@makefnmark}{\hbox to \z@{$^{\@thefnmark}$\hss}}
+ % The footnote-mark was overlapping the footnote-text,
+ % added the following to fix this problem (MK)
+ \long\def\@makefntext##1{%
+ \parindent 1em\noindent
+ \hbox to 1.8em{\hss $\m@th ^{\@thefnmark}$}##1
+ }
+ \thispagestyle{empty}
+ \@maketitle
+ \@thanks
+ \@notice
+ \endgroup
+ \let\maketitle\relax
+ \let\thanks\relax
+}
+
+% rules for title box at top of first page
+\newcommand{\@toptitlebar}{
+ \hrule height 4\p@
+ \vskip 0.25in
+ \vskip -\parskip%
+}
+\newcommand{\@bottomtitlebar}{
+ \vskip 0.29in
+ \vskip -\parskip
+ \hrule height 1\p@
+ \vskip 0.09in%
+}
+
+% create title (includes both anonymized and non-anonymized versions)
+\providecommand{\@maketitle}{}
+\renewcommand{\@maketitle}{%
+ \vbox{%
+ \hsize\textwidth
+ \linewidth\hsize
+ \vskip 0.1in
+ \@toptitlebar
+ \centering
+ {\LARGE\bf \@title\par}
+ \@bottomtitlebar
+ \if@submission
+ \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}
+ Anonymous Author(s) \\
+ Affiliation \\
+ Address \\
+ \texttt{email} \\
+ \end{tabular}%
+ \else
+ \def\And{%
+ \end{tabular}\hfil\linebreak[0]\hfil%
+ \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces%
+ }
+ \def\AND{%
+ \end{tabular}\hfil\linebreak[4]\hfil%
+ \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces%
+ }
+ \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\@author\end{tabular}%
+ \fi
+ \vskip 0.3in \@minus 0.1in
+ }
+}
+
+% add conference notice to bottom of first page
+\newcommand{\ftype@noticebox}{8}
+\newcommand{\@notice}{%
+ % give a bit of extra room back to authors on first page
+ \enlargethispage{2\baselineskip}%
+ \@float{noticebox}[b]%
+ \footnotesize\@noticestring%
+ \end@float%
+}
+
+% abstract styling
+\renewenvironment{abstract}%
+{%
+ \vskip 0.075in%
+ \centerline%
+ {\large\bf Abstract}%
+ \vspace{0.5ex}%
+ \begin{quote}%
+}
+{
+ \par%
+ \end{quote}%
+ \vskip 1ex%
+}
+
+% For the paper checklist
+\newcommand{\answerYes}[1][]{\textcolor{blue}{[Yes] #1}}
+\newcommand{\answerNo}[1][]{\textcolor{orange}{[No] #1}}
+\newcommand{\answerNA}[1][]{\textcolor{gray}{[N/A] #1}}
+\newcommand{\answerTODO}[1][]{\textcolor{red}{\bf [TODO]}}
+
+% handle tweaks for camera-ready copy vs. submission copy
+\if@preprint
+ \newcommand{\@noticestring}{%
+ Preprint. Under review.%
+ }
+\else
+ \if@neuripsfinal
+ \newcommand{\@noticestring}{%
+ \@neuripsordinal\/ Conference on Neural Information Processing Systems
+ (NeurIPS \@neuripsyear).%, \@neuripslocation.%
+ }
+ \else
+ \newcommand{\@noticestring}{%
+ Submitted to \@neuripsordinal\/ Conference on Neural Information
+ Processing Systems (NeurIPS \@neuripsyear). Do not distribute.%
+ }
+
+ % hide the acknowledgements
+ \NewEnviron{hide}{}
+ \let\ack\hide
+ \let\endack\endhide
+
+ % line numbers for submission
+ \RequirePackage{lineno}
+ \linenumbers
+
+ % fix incompatibilities between lineno and amsmath, if required, by
+ % transparently wrapping linenomath environments around amsmath
+ % environments
+ \AtBeginDocument{%
+ \@ifpackageloaded{amsmath}{%
+ \newcommand*\patchAmsMathEnvironmentForLineno[1]{%
+ \expandafter\let\csname old#1\expandafter\endcsname\csname #1\endcsname
+ \expandafter\let\csname oldend#1\expandafter\endcsname\csname end#1\endcsname
+ \renewenvironment{#1}%
+ {\linenomath\csname old#1\endcsname}%
+ {\csname oldend#1\endcsname\endlinenomath}%
+ }%
+ \newcommand*\patchBothAmsMathEnvironmentsForLineno[1]{%
+ \patchAmsMathEnvironmentForLineno{#1}%
+ \patchAmsMathEnvironmentForLineno{#1*}%
+ }%
+ \patchBothAmsMathEnvironmentsForLineno{equation}%
+ \patchBothAmsMathEnvironmentsForLineno{align}%
+ \patchBothAmsMathEnvironmentsForLineno{flalign}%
+ \patchBothAmsMathEnvironmentsForLineno{alignat}%
+ \patchBothAmsMathEnvironmentsForLineno{gather}%
+ \patchBothAmsMathEnvironmentsForLineno{multline}%
+ }
+ {}
+ }
+ \fi
+\fi
+
+
+\endinput
diff --git a/paper/paper.aux b/paper/paper.aux
new file mode 100644
index 0000000000000000000000000000000000000000..dcee3224cb2d4c58ebb21496981072d072731cc7
--- /dev/null
+++ b/paper/paper.aux
@@ -0,0 +1,40 @@
+\relax
+\providecommand\hyper@newdestlabel[2]{}
+\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
+\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
+\global\let\oldnewlabel\newlabel
+\gdef\newlabel#1#2{\newlabelxx{#1}#2}
+\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
+\AtEndDocument{\ifx\hyper@anchor\@undefined
+\let\newlabel\oldnewlabel
+\fi}
+\fi}
+\global\let\hyper@last\relax
+\gdef\HyperFirstAtBeginDocument#1{#1}
+\providecommand\HyField@AuxAddToFields[1]{}
+\providecommand\HyField@AuxAddToCoFields[2]{}
+\@writefile{toc}{\contentsline {section}{\numberline {1}Submission of papers to NeurIPS 2022}{1}{section.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsection}{\numberline {1.1}Style}{1}{subsection.1.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsection}{\numberline {1.2}Retrieval of style files}{1}{subsection.1.2}\protected@file@percent }
+\@writefile{toc}{\contentsline {paragraph}{Preprint option}{1}{section*.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {section}{\numberline {2}General formatting instructions}{2}{section.2}\protected@file@percent }
+\newlabel{gen_inst}{{2}{2}{General formatting instructions}{section.2}{}}
+\@writefile{toc}{\contentsline {section}{\numberline {3}Headings: first level}{2}{section.3}\protected@file@percent }
+\newlabel{headings}{{3}{2}{Headings: first level}{section.3}{}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Headings: second level}{2}{subsection.3.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.1.1}Headings: third level}{2}{subsubsection.3.1.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {paragraph}{Paragraphs}{2}{section*.2}\protected@file@percent }
+\@writefile{toc}{\contentsline {section}{\numberline {4}Citations, figures, tables, references}{2}{section.4}\protected@file@percent }
+\newlabel{others}{{4}{2}{Citations, figures, tables, references}{section.4}{}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Citations within the text}{2}{subsection.4.1}\protected@file@percent }
+\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Sample figure caption.}}{3}{figure.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Footnotes}{3}{subsection.4.2}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Figures}{3}{subsection.4.3}\protected@file@percent }
+\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Sample table title}}{4}{table.1}\protected@file@percent }
+\newlabel{sample-table}{{1}{4}{Sample table title}{table.1}{}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.4}Tables}{4}{subsection.4.4}\protected@file@percent }
+\@writefile{toc}{\contentsline {section}{\numberline {5}Final instructions}{4}{section.5}\protected@file@percent }
+\@writefile{toc}{\contentsline {section}{\numberline {6}Preparing PDF files}{4}{section.6}\protected@file@percent }
+\@writefile{toc}{\contentsline {subsection}{\numberline {6.1}Margins in \LaTeX {}}{5}{subsection.6.1}\protected@file@percent }
+\@writefile{toc}{\contentsline {section}{\numberline {A}Appendix}{6}{appendix.A}\protected@file@percent }
+\gdef \@abspage@last{6}
diff --git a/paper/paper.fdb_latexmk b/paper/paper.fdb_latexmk
new file mode 100644
index 0000000000000000000000000000000000000000..fa954fc439ff45b750ed23ac66cced7e48655a2d
--- /dev/null
+++ b/paper/paper.fdb_latexmk
@@ -0,0 +1,123 @@
+# Fdb version 4
+["pdflatex"] 1679380233 "/Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper/paper.tex" "paper.pdf" "paper" 1679380235 0
+ "/Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper/paper.tex" 1679380074 20402 03963733f316ff58f73e6c8697a72e22 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/base/8r.enc" 1165713224 4850 80dc9bab7f31fb78a000ccfed0e27cab ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/cm-super/cm-super-t1.enc" 1136849721 2971 def0b6c1f0b107b3b936def894055589 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/map/fontname/texfonts.map" 1577235249 3524 cb3e574dea2d1052e39280babc910dc8 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/helvetic/phvr8r.tfm" 1136768653 4712 9ef4d7d106579d4b136e1529e1a4533c ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/helvetic/phvr8t.tfm" 1136768653 7040 b2bd27e2bfe6f6948cbc3239cae7444f ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm" 1136768653 4524 6bce29db5bc272ba5f332261583fee9c ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm" 1136768653 6880 f19b8995b61c334d78fc734065f6b4d4 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8c.tfm" 1136768653 1352 fa28a7e6d323c65ce7d13d5342ff6be2 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8r.tfm" 1136768653 4408 25b74d011a4c66b7f212c0cc3c90061b ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm" 1136768653 6672 e3ab9e37e925f3045c9005e6d1473d56 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8r.tfm" 1136768653 4640 532ca3305aad10cc01d769f3f91f1029 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8t.tfm" 1136768653 6944 94c55ad86e6ea2826f78ba2240d50df9 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/jknappen/ec/ectt1000.tfm" 1136768653 1536 06717a2b50de47d4087ac0e6cd759455 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm" 1246382020 1004 54797486969f23fa377b128694d548df ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex9.tfm" 1246382020 996 a18840b13b499c08ac2de96a99eda4bc ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm" 1246382020 916 f87d7c45f9c908e672703b83b72241a3 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam5.tfm" 1246382020 924 9904cf1d39e9767e7a3622f2a125a565 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam7.tfm" 1246382020 928 2dc8d444221b7a635bb58038579b861a ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm" 1246382020 908 2921f8a10601f252058503cc6570e581 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm5.tfm" 1246382020 940 75ac932a52f80982a9f8ea75d03a34cf ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm7.tfm" 1246382020 940 228d6584342e91276bf566bcf9716b83 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi6.tfm" 1136768653 1512 f21f83efb36853c0b70002322c1ab3ad ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi9.tfm" 1136768653 1524 d89e2d087a9828407a196f428428ef4a ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr6.tfm" 1136768653 1300 b62933e007d01cfd073f79b963c01526 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr9.tfm" 1136768653 1292 6b21b9c2c7bebb38aa2273f7ca0fb3af ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy6.tfm" 1136768653 1116 933a60c408fc0a863a92debe84b2d294 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy9.tfm" 1136768653 1116 25a7bf822c58caf309a702ef79f4afbb ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb" 1248133631 36299 5f9df58c2139e7edcf37c8fca4bd384d ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb" 1248133631 35752 024fb6c41858982481f6968b5fc26508 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr7.pfb" 1248133631 32762 224316ccc9ad3ca0423a14971cfa7fc1 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb" 1248133631 32569 5e5ddc8df908dea60932f3c484a54c0d ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/symbols/msbm10.pfb" 1248133631 34694 ad62b13721ee8eda1dcc8993c8bd7041 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/public/cm-super/sftt1000.pfb" 1215737283 169201 9ebf99020dde51a5086e186761a34e8f ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/urw/helvetic/uhvr8a.pfb" 1136849748 44648 23115b2a545ebfe2c526c3ca99db8b95 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmb8a.pfb" 1136849748 44729 811d6c62865936705a31c797a1d5dada ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmr8a.pfb" 1136849748 46026 6dab18b61c907687b520c72847215a68 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmri8a.pfb" 1136849748 45458 a3faba884469519614ca56ba5f6b1de1 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/helvetic/phvr8t.vf" 1136768653 2344 44ff28c9ef2fc97180cd884f900fee71 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf" 1136768653 2340 df9c920cc5688ebbf16a93f45ce7bdd3 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8c.vf" 1136768653 3556 8a9a6dcbcd146ef985683f677f4758a6 ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8t.vf" 1136768653 2348 91706c542228501c410c266421fbe30c ""
+ "/usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmri8t.vf" 1136768653 2328 6cd7df782b09b29cfc4d93e55b6b9a59 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii" 1461363279 71627 94eb9990bed73c364d7f53f960cc8c5b ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty" 1575674566 24708 5584a51a7101caf7e6bbf1fc27d8f7b1 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty" 1576625341 40635 c40361e206be584d448876bba8a64a3b ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty" 1576016050 33961 6b5c75130e435b2bfdb9f480a09a39f9 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty" 1576625273 7734 b98cbb34c81f667027c1e3ebdbfce34b ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty" 1576625223 8371 9d55b8bd010bc717624922fb3477d92e ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty" 1644112042 7237 bdd120a32c8fdb4b433cf9ca2e7cd98a ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty" 1572645307 1057 525c2192b5febbd8c1f662c9468335bb ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty" 1575499628 8356 7bbb2c2373aa810be568c29e333da8ed ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty" 1576625065 31769 002a487f55041f8e805cfbf6385ffd97 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty" 1576878844 5412 d5a2436094cd7be85769db90f29250a6 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty" 1600895880 17859 4409f8f50cd365c68e684407e5350b1b ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty" 1576015897 19007 15924f7228aca6c6d184b115f4baa231 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty" 1593379760 20089 80423eac55aa175305d35b49e04fe23b ""
+ "/usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty" 1576624663 7008 f92eaa0a3872ed622bbf538217cd2ab7 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty" 1359763108 5949 3f3fd50a8cc94c3d4cbf4fc66cd3df1c ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd" 1359763108 961 6518c6525a34feb5e8250ffa91731cff ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd" 1359763108 961 d02606146ba5601b5645f987c92e6193 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty" 1576191570 19336 ce7ae9438967282886b3b036cfad1e4d ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty" 1576625391 3935 57aa3c3e203a5c2effb4d2bd2efbc323 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls" 1667332637 20144 d5ecf0a5140c8d8d8b72cbe86e320eff ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty" 1667332637 3052 30236f0cc243a8651b82240dfd2e8b9d ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty" 1667332637 2462 8ce5f9a9c63002f2c1af03c262cf29af ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty" 1654720880 5119 4ce42f43368f652f9c9522d943cce8e4 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty" 1654720880 5319 48d7f3cfa322abd2788e3c09d624b922 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty" 1654720880 5048 84b05796b49b69e2d4257d537721c960 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo" 1667332637 8448 c33a4e1cb35cee9b33c2b21033b73e39 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd" 1667332637 2443 f9c272838e65ac147422f32fe87b76a1 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty" 1579038678 6078 f1cb470c9199e7110a27851508ed7a5c ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty" 1399239813 4378 f429f0da968c278653359293040a8f52 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty" 1601931149 46845 3b58f70c6e861a13d927bff09d35ecbc ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty" 1578002852 41601 9cf6c5257b1bc7af01a58859749dd37a ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg" 1459978653 1213 620bba36b25224fa9b7e1ccb4ecb76fd ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def" 1663965824 19448 1e988b341dda20961a6b931bcde55519 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty" 1654720880 2671 7e67d78d9b88c845599a85b2d41f2e39 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx" 1667332637 2885 9c645d672ae17285bba324998918efd8 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty" 1580250785 17914 4c28a13fc3d975e6e81c9bea1d697276 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def" 1675889938 48272 99ede602a8ace626d8ed02f058a4bf8e ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty" 1675889938 223129 4edf043af471f3251c66e432cfa22987 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty" 1675889938 12947 1ce831528e963a8568de1f4d67cfb982 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def" 1675889938 14249 d947c5c09f3af04ae2f37fc11c7ac2f6 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def" 1675889938 117125 aa115cac3914abcf3769f370e6325117 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty" 1655478651 22555 6d8e155cfef6d82c3d5c742fea7c992e ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty" 1665067230 13815 760b0c02f691ea230f5359c4e1de23a7 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def" 1673989714 30429 213676d4c7327a21d91ddaed900e7b81 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty" 1575499565 5766 13a9e8766c47f30327caf893ece86ac8 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty" 1674162155 152262 eb3bf7a579d4a34a0ed4f6b445bafbd8 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def" 1678741534 48246 c3eed060aba663f58af3ff756e83f2bd ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg" 1678741534 26842 05a01d67d23e805520393a049533b8c0 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty" 1678741534 98531 f79ec363f3014ada2cb766715926ecc9 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg" 1678741534 22906 7a8d9f8766e60f694cdde7648421c2ab ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg" 1678741534 5929 0e1d31c98c10fece90f470d5746ecdd6 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg" 1678741534 5594 45ca1ba048c2fa6267d5419cc463d804 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg" 1678741534 12427 02d1873769280609ae2509e963f1fa66 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty" 1291685959 45456 1c8843383c0bd05870c45fa0ebea6cc2 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd" 1586716065 1483 47067fbe7c3ffed1ede7aaa7b8549d7a ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd" 1137110629 774 61d7da1e9f9e74989b196d147e623736 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd" 1137110629 619 96f56dc5d1ef1fe1121f1cfeec70ee0c ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty" 1576624809 9878 9e94e8fa600d95f9c7731bb21dfb67a4 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty" 1657483315 9714 ba3194bd52c8499b3f1e3eb91d409670 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty" 1253232110 1380 971a51b00a14503ddf754cab24c3f209 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty" 1137111039 4029 0462ee5ab265cf59dc15a41a3b883101 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty" 1388531844 12796 8edb7d69a20b857904dd0ea757c14ec9 ""
+ "/usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty" 1655066402 56148 51a9a8571c07b9921892ae11063ae853 ""
+ "/usr/local/texlive/2023/texmf-dist/web2c/texmf.cnf" 1677365944 40745 87bb86a62d462f93a1efc6b0c65c872e ""
+ "/usr/local/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map" 1679378061 4623355 b8cad38412faebe1ce5b780a45db67bd ""
+ "/usr/local/texlive/2023/texmf-var/web2c/pdftex/pdflatex.fmt" 1679378051 7883457 1aa397298814c10ba7469ca52ed0fc97 ""
+ "/usr/local/texlive/2023/texmf.cnf" 1678822455 577 1b2b1af17af2508fb20dd85783af4050 ""
+ "neurips_2022.sty" 1679380107 11562 bed3b2b3dd41efa3f6cf9e27f533a20e ""
+ "paper.aux" 1679380235 3253 f4fa39f137bc0598f7069980635fef48 "pdflatex"
+ "paper.out" 1679380235 2443 adc4f3226fc4bad2b7e73b5f36623ad6 "pdflatex"
+ "paper.tex" 1679380074 20402 03963733f316ff58f73e6c8697a72e22 ""
+ (generated)
+ "paper.aux"
+ "paper.log"
+ "paper.out"
+ "paper.pdf"
+ (rewritten before read)
diff --git a/paper/paper.fls b/paper/paper.fls
new file mode 100644
index 0000000000000000000000000000000000000000..282641dd80fe0ab23c348048f4ac8780a5253bdd
--- /dev/null
+++ b/paper/paper.fls
@@ -0,0 +1,692 @@
+PWD /Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper
+INPUT /usr/local/texlive/2023/texmf.cnf
+INPUT /usr/local/texlive/2023/texmf-dist/web2c/texmf.cnf
+INPUT /usr/local/texlive/2023/texmf-var/web2c/pdftex/pdflatex.fmt
+INPUT /Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper/paper.tex
+OUTPUT paper.log
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo
+INPUT ./neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT neurips_2022.sty
+INPUT ./neurips_2022.sty
+INPUT neurips_2022.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/map/fontname/texfonts.map
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/atbegshi/atbegshi.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/atveryend/atveryend.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+INPUT ./paper.aux
+INPUT paper.aux
+INPUT paper.aux
+OUTPUT paper.aux
+INPUT ./paper.out
+INPUT paper.out
+INPUT ./paper.out
+INPUT paper.out
+INPUT ./paper.out
+INPUT paper.out
+INPUT ./paper.out
+INPUT paper.out
+OUTPUT paper.pdf
+INPUT ./paper.out
+INPUT ./paper.out
+OUTPUT paper.out
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+INPUT /usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+INPUT /usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+INPUT /usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam5.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm5.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/jknappen/ec/ectt1000.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/helvetic/phvr8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm
+INPUT /usr/local/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/base/8r.enc
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/cm-super/cm-super-t1.enc
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/helvetic/phvr8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/helvetic/phvr8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmri8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr9.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr6.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi9.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi6.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy9.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy6.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex9.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm7.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmr8c.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmr8c.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8t.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmri8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmri8r.tfm
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/vf/adobe/times/ptmb8t.vf
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/tfm/adobe/times/ptmb8r.tfm
+INPUT paper.aux
+INPUT ./paper.out
+INPUT ./paper.out
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr7.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/symbols/msbm10.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/public/cm-super/sftt1000.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/urw/helvetic/uhvr8a.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmb8a.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmr8a.pfb
+INPUT /usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmri8a.pfb
diff --git a/paper/paper.log b/paper/paper.log
new file mode 100644
index 0000000000000000000000000000000000000000..e427058637c174fd0a44f802ace00fdf9bcfa837
--- /dev/null
+++ b/paper/paper.log
@@ -0,0 +1,460 @@
+This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023) (preloaded format=pdflatex 2023.3.21) 21 MAR 2023 07:30
+entering extended mode
+ restricted \write18 enabled.
+ file:line:error style messages enabled.
+ %&-line parsing enabled.
+**/Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper/paper.tex
+(/Users/patrickaltmeyer/code/conformal-counterfactual-explanations/paper/paper.tex
+LaTeX2e <2022-11-01> patch level 1
+L3 programming layer <2023-02-22> (/usr/local/texlive/2023/texmf-dist/tex/latex/base/article.cls
+Document Class: article 2022/07/02 v1.4n Standard LaTeX document class
+(/usr/local/texlive/2023/texmf-dist/tex/latex/base/size10.clo
+File: size10.clo 2022/07/02 v1.4n Standard LaTeX file (size option)
+)
+\c@part=\count185
+\c@section=\count186
+\c@subsection=\count187
+\c@subsubsection=\count188
+\c@paragraph=\count189
+\c@subparagraph=\count190
+\c@figure=\count191
+\c@table=\count192
+\abovecaptionskip=\skip48
+\belowcaptionskip=\skip49
+\bibindent=\dimen140
+) (./neurips_2022.sty
+Package: neurips_2022 2022/03/31 NeurIPS 2022 submission/camera-ready style file
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/environ/environ.sty
+Package: environ 2014/05/04 v0.3 A new way to define environments
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/trimspaces/trimspaces.sty
+Package: trimspaces 2009/09/17 v1.1 Trim spaces around a token list
+)
+\@envbody=\toks16
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/natbib/natbib.sty
+Package: natbib 2010/09/13 8.31b (PWD, AO)
+\bibhang=\skip50
+\bibsep=\skip51
+LaTeX Info: Redefining \cite on input line 694.
+\c@NAT@ctr=\count193
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/geometry/geometry.sty
+Package: geometry 2020/01/02 v5.9 Page Geometry
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty
+Package: keyval 2022/05/29 v1.15 key=value parser (DPC)
+\KV@toks@=\toks17
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/iftex/ifvtex.sty
+Package: ifvtex 2019/10/25 v1.7 ifvtex legacy package. Use iftex instead.
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/iftex/iftex.sty
+Package: iftex 2022/02/03 v1.0f TeX engine tests
+))
+\Gm@cnth=\count194
+\Gm@cntv=\count195
+\c@Gm@tempcnt=\count196
+\Gm@bindingoffset=\dimen141
+\Gm@wd@mp=\dimen142
+\Gm@odd@mp=\dimen143
+\Gm@even@mp=\dimen144
+\Gm@layoutwidth=\dimen145
+\Gm@layoutheight=\dimen146
+\Gm@layouthoffset=\dimen147
+\Gm@layoutvoffset=\dimen148
+\Gm@dimlist=\toks18
+)
+\@neuripsabovecaptionskip=\skip52
+\@neuripsbelowcaptionskip=\skip53
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/lineno/lineno.sty
+Package: lineno 2023/01/19 line numbers on paragraphs v5.1
+\linenopenalty=\count197
+\output=\toks19
+\linenoprevgraf=\count198
+\linenumbersep=\dimen149
+\linenumberwidth=\dimen150
+\c@linenumber=\count199
+\c@pagewiselinenumber=\count266
+\c@LN@truepage=\count267
+\c@internallinenumber=\count268
+\c@internallinenumbers=\count269
+\quotelinenumbersep=\dimen151
+\bframerule=\dimen152
+\bframesep=\dimen153
+\bframebox=\box51
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty
+Package: etoolbox 2020/10/05 v2.5k e-TeX tools for LaTeX (JAW)
+\etb@tempcnta=\count270
+)
+LaTeX Info: Redefining \\ on input line 3131.
+)) (/usr/local/texlive/2023/texmf-dist/tex/latex/base/inputenc.sty
+Package: inputenc 2021/02/14 v1.3d Input encoding file
+\inpenc@prehook=\toks20
+\inpenc@posthook=\toks21
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/base/fontenc.sty
+Package: fontenc 2021/04/29 v2.0v Standard LaTeX package
+LaTeX Font Info: Trying to load font information for T1+ptm on input line 112.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1ptm.fd
+File: t1ptm.fd 2001/06/04 font definitions for T1/ptm.
+)) (/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hyperref.sty
+Package: hyperref 2023-02-07 v7.00v Hypertext links for LaTeX
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
+Package: ltxcmds 2020-05-10 v1.25 LaTeX kernel commands for general use (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
+Package: pdftexcmds 2020-06-27 v0.33 Utility functions of pdfTeX for LuaTeX (HO)
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/infwarerr/infwarerr.sty
+Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO)
+)
+Package pdftexcmds Info: \pdf@primitive is available.
+Package pdftexcmds Info: \pdf@ifprimitive is available.
+Package pdftexcmds Info: \pdfdraftmode found.
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
+Package: kvsetkeys 2022-10-05 v1.19 Key value parser (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
+Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/pdfescape/pdfescape.sty
+Package: pdfescape 2019/12/09 v1.15 Implements pdfTeX's escape features (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/hycolor/hycolor.sty
+Package: hycolor 2020-01-27 v1.10 Color options for hyperref/bookmark (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/letltxmacro/letltxmacro.sty
+Package: letltxmacro 2019/12/03 v1.6 Let assignment for LaTeX macros (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/auxhook/auxhook.sty
+Package: auxhook 2019-12-17 v1.6 Hooks for auxiliary files (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/nameref.sty
+Package: nameref 2022-05-17 v2.50 Cross-referencing by name of section
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/refcount/refcount.sty
+Package: refcount 2019/12/15 v3.6 Data extraction from label references (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
+Package: gettitlestring 2019/12/15 v1.6 Cleanup title references (HO)
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty
+Package: kvoptions 2022-06-15 v3.15 Key value format for package options (HO)
+))
+\c@section@level=\count271
+)
+\@linkdim=\dimen154
+\Hy@linkcounter=\count272
+\Hy@pagecounter=\count273
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/pd1enc.def
+File: pd1enc.def 2023-02-07 v7.00v Hyperref: PDFDocEncoding definition (HO)
+Now handling font encoding PD1 ...
+... no UTF-8 mapping file for font encoding PD1
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/intcalc/intcalc.sty
+Package: intcalc 2019/12/15 v1.3 Expandable calculations with integers (HO)
+) (/usr/local/texlive/2023/texmf-dist/tex/generic/etexcmds/etexcmds.sty
+Package: etexcmds 2019/12/15 v1.7 Avoid name clashes with e-TeX commands (HO)
+)
+\Hy@SavedSpaceFactor=\count274
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/puenc.def
+File: puenc.def 2023-02-07 v7.00v Hyperref: PDF Unicode definition (HO)
+Now handling font encoding PU ...
+... no UTF-8 mapping file for font encoding PU
+)
+Package hyperref Info: Hyper figures OFF on input line 4177.
+Package hyperref Info: Link nesting OFF on input line 4182.
+Package hyperref Info: Hyper index ON on input line 4185.
+Package hyperref Info: Plain pages OFF on input line 4192.
+Package hyperref Info: Backreferencing OFF on input line 4197.
+Package hyperref Info: Implicit mode ON; LaTeX internals redefined.
+Package hyperref Info: Bookmarks ON on input line 4425.
+\c@Hy@tempcnt=\count275
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/url/url.sty
+\Urlmuskip=\muskip16
+Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc.
+)
+LaTeX Info: Redefining \url on input line 4763.
+\XeTeXLinkMargin=\dimen155
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/bitset/bitset.sty
+Package: bitset 2019/12/09 v1.3 Handle bit-vector datatype (HO)
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
+Package: bigintcalc 2019/12/15 v1.5 Expandable calculations on big integers (HO)
+))
+\Fld@menulength=\count276
+\Field@Width=\dimen156
+\Fld@charsize=\dimen157
+Package hyperref Info: Hyper figures OFF on input line 6042.
+Package hyperref Info: Link nesting OFF on input line 6047.
+Package hyperref Info: Hyper index ON on input line 6050.
+Package hyperref Info: backreferencing OFF on input line 6057.
+Package hyperref Info: Link coloring OFF on input line 6062.
+Package hyperref Info: Link coloring with OCG OFF on input line 6067.
+Package hyperref Info: PDF/A mode OFF on input line 6072.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/base/atbegshi-ltx.sty
+Package: atbegshi-ltx 2021/01/10 v1.0c Emulation of the original atbegshi
+package with kernel methods
+)
+\Hy@abspage=\count277
+\c@Item=\count278
+\c@Hfootnote=\count279
+)
+Package hyperref Info: Driver (autodetected): hpdftex.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/hyperref/hpdftex.def
+File: hpdftex.def 2023-02-07 v7.00v Hyperref driver for pdfTeX
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/base/atveryend-ltx.sty
+Package: atveryend-ltx 2020/08/19 v1.0a Emulation of the original atveryend package
+with kernel methods
+)
+\Fld@listcount=\count280
+\c@bookmark@seq@number=\count281
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
+Package: rerunfilecheck 2022-07-10 v1.10 Rerun checks for auxiliary files (HO)
+ (/usr/local/texlive/2023/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
+Package: uniquecounter 2019/12/15 v1.4 Provide unlimited unique counter (HO)
+)
+Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 285.
+)
+\Hy@SectionHShift=\skip54
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/booktabs/booktabs.sty
+Package: booktabs 2020/01/12 v1.61803398 Publication quality tables
+\heavyrulewidth=\dimen158
+\lightrulewidth=\dimen159
+\cmidrulewidth=\dimen160
+\belowrulesep=\dimen161
+\belowbottomsep=\dimen162
+\aboverulesep=\dimen163
+\abovetopsep=\dimen164
+\cmidrulesep=\dimen165
+\cmidrulekern=\dimen166
+\defaultaddspace=\dimen167
+\@cmidla=\count282
+\@cmidlb=\count283
+\@aboverulesep=\dimen168
+\@belowrulesep=\dimen169
+\@thisruleclass=\count284
+\@lastruleclass=\count285
+\@thisrulewidth=\dimen170
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty
+Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support
+\@emptytoks=\toks22
+\symAMSa=\mathgroup4
+\symAMSb=\mathgroup5
+LaTeX Font Info: Redeclaring math symbol \hbar on input line 98.
+LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold'
+(Font) U/euf/m/n --> U/euf/b/n on input line 106.
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/units/nicefrac.sty
+Package: nicefrac 1998/08/04 v0.9b Nice fractions
+\L@UnitsRaiseDisplaystyle=\skip55
+\L@UnitsRaiseTextstyle=\skip56
+\L@UnitsRaiseScriptstyle=\skip57
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/base/ifthen.sty
+Package: ifthen 2022/04/13 v1.1d Standard LaTeX ifthen package (DPC)
+)) (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.sty
+Package: microtype 2023/03/13 v3.1a Micro-typographical refinements (RS)
+\MT@toks=\toks23
+\MT@tempbox=\box52
+\MT@count=\count286
+LaTeX Info: Redefining \noprotrusionifhmode on input line 1059.
+LaTeX Info: Redefining \leftprotrusion on input line 1060.
+\MT@prot@toks=\toks24
+LaTeX Info: Redefining \rightprotrusion on input line 1078.
+LaTeX Info: Redefining \textls on input line 1368.
+\MT@outer@kern=\dimen171
+LaTeX Info: Redefining \textmicrotypecontext on input line 1988.
+\MT@listname@count=\count287
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype-pdftex.def
+File: microtype-pdftex.def 2023/03/13 v3.1a Definitions specific to pdftex (RS)
+LaTeX Info: Redefining \lsstyle on input line 902.
+LaTeX Info: Redefining \lslig on input line 902.
+\MT@outer@space=\skip58
+)
+Package microtype Info: Loading configuration file microtype.cfg.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/microtype.cfg
+File: microtype.cfg 2023/03/13 v3.1a microtype main configuration file (RS)
+)) (/usr/local/texlive/2023/texmf-dist/tex/latex/xcolor/xcolor.sty
+Package: xcolor 2022/06/12 v2.14 LaTeX color extensions (UK)
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/graphics-cfg/color.cfg
+File: color.cfg 2016/01/02 v1.6 sample color configuration
+)
+Package xcolor Info: Driver file: pdftex.def on input line 227.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def
+File: pdftex.def 2022/09/22 v1.2b Graphics/color driver for pdftex
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/graphics/mathcolor.ltx)
+Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1353.
+Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1357.
+Package xcolor Info: Model `RGB' extended on input line 1369.
+Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1371.
+Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1372.
+Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1373.
+Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1374.
+Package xcolor Info: Model `Gray' substituted by `gray' on input line 1375.
+Package xcolor Info: Model `wave' substituted by `hsb' on input line 1376.
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
+File: l3backend-pdftex.def 2023-01-16 L3 backend support: PDF output (pdfTeX)
+\l__color_backend_stack_int=\count288
+\l__pdf_internal_box=\box53
+) (./paper.aux)
+\openout1 = `paper.aux'.
+
+LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+LaTeX Font Info: Checking defaults for PU/pdf/m/n on input line 81.
+LaTeX Font Info: ... okay on input line 81.
+
+*geometry* driver: auto-detecting
+*geometry* detected driver: pdftex
+*geometry* verbose mode - [ preamble ] result:
+* driver: pdftex
+* paper: letterpaper
+* layout: <same size as paper>
+* layoutoffset:(h,v)=(0.0pt,0.0pt)
+* modes:
+* h-part:(L,W,R)=(92.14519pt, 430.00462pt, 92.14519pt)
+* v-part:(T,H,B)=(95.39737pt, 556.47656pt, 143.09605pt)
+* \paperwidth=614.295pt
+* \paperheight=794.96999pt
+* \textwidth=430.00462pt
+* \textheight=556.47656pt
+* \oddsidemargin=19.8752pt
+* \evensidemargin=19.8752pt
+* \topmargin=-13.87262pt
+* \headheight=12.0pt
+* \headsep=25.0pt
+* \topskip=10.0pt
+* \footskip=30.0pt
+* \marginparwidth=65.0pt
+* \marginparsep=11.0pt
+* \columnsep=10.0pt
+* \skip\footins=9.0pt plus 4.0pt minus 2.0pt
+* \hoffset=0.0pt
+* \voffset=0.0pt
+* \mag=1000
+* \@twocolumnfalse
+* \@twosidefalse
+* \@mparswitchfalse
+* \@reversemarginfalse
+* (1in=72.27pt=25.4mm, 1cm=28.453pt)
+
+*geometry* verbose mode - [ newgeometry ] result:
+* driver: pdftex
+* paper: letterpaper
+* layout: <same size as paper>
+* layoutoffset:(h,v)=(0.0pt,0.0pt)
+* modes:
+* h-part:(L,W,R)=(108.405pt, 397.48499pt, 108.40501pt)
+* v-part:(T,H,B)=(72.26999pt, 650.43pt, 72.27pt)
+* \paperwidth=614.295pt
+* \paperheight=794.96999pt
+* \textwidth=397.48499pt
+* \textheight=650.43pt
+* \oddsidemargin=36.13501pt
+* \evensidemargin=36.13501pt
+* \topmargin=-37.0pt
+* \headheight=12.0pt
+* \headsep=25.0pt
+* \topskip=10.0pt
+* \footskip=30.0pt
+* \marginparwidth=65.0pt
+* \marginparsep=11.0pt
+* \columnsep=10.0pt
+* \skip\footins=9.0pt plus 4.0pt minus 2.0pt
+* \hoffset=0.0pt
+* \voffset=0.0pt
+* \mag=1000
+* \@twocolumnfalse
+* \@twosidefalse
+* \@mparswitchfalse
+* \@reversemarginfalse
+* (1in=72.27pt=25.4mm, 1cm=28.453pt)
+
+Package hyperref Info: Link coloring OFF on input line 81.
+(./paper.out) (./paper.out)
+\@outlinefile=\write3
+\openout3 = `paper.out'.
+
+LaTeX Info: Redefining \microtypecontext on input line 81.
+Package microtype Info: Applying patch `item' on input line 81.
+Package microtype Info: Applying patch `toc' on input line 81.
+Package microtype Info: Applying patch `eqnum' on input line 81.
+Package microtype Info: Applying patch `footnote' on input line 81.
+Package microtype Info: Applying patch `verbatim' on input line 81.
+Package microtype Info: Generating PDF output.
+Package microtype Info: Character protrusion enabled (level 2).
+Package microtype Info: Using default protrusion set `alltext'.
+Package microtype Info: Automatic font expansion enabled (level 2),
+(microtype) stretch: 20, shrink: 20, step: 1, non-selected.
+Package microtype Info: Using default expansion set `alltext-nott'.
+LaTeX Info: Redefining \showhyphens on input line 81.
+Package microtype Info: No adjustment of tracking.
+Package microtype Info: No adjustment of interword spacing.
+Package microtype Info: No adjustment of character kerning.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-ptm.cfg
+File: mt-ptm.cfg 2006/04/20 v1.7 microtype config. file: Times (RS)
+) (/usr/local/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+[Loading MPS to PDF converter (version 2006.09.02).]
+\scratchcounter=\count289
+\scratchdimen=\dimen172
+\scratchbox=\box54
+\nofMPsegments=\count290
+\nofMParguments=\count291
+\everyMPshowfont=\toks25
+\MPscratchCnt=\count292
+\MPscratchDim=\dimen173
+\MPnumerator=\count293
+\makeMPintoPDFobject=\count294
+\everyMPtoPDFconversion=\toks26
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-cmr.cfg
+File: mt-cmr.cfg 2013/05/19 v2.2 microtype config. file: Computer Modern Roman (RS)
+)
+LaTeX Font Info: Trying to load font information for U+msa on input line 85.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd
+File: umsa.fd 2013/01/14 v3.01 AMS symbols A
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msa.cfg
+File: mt-msa.cfg 2006/02/04 v1.1 microtype config. file: AMS symbols (a) (RS)
+)
+LaTeX Font Info: Trying to load font information for U+msb on input line 85.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd
+File: umsb.fd 2013/01/14 v3.01 AMS symbols B
+) (/usr/local/texlive/2023/texmf-dist/tex/latex/microtype/mt-msb.cfg
+File: mt-msb.cfg 2005/06/01 v1.0 microtype config. file: AMS symbols (b) (RS)
+)
+LaTeX Font Info: Trying to load font information for T1+cmtt on input line 85.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/base/t1cmtt.fd
+File: t1cmtt.fd 2022/07/10 v2.5l Standard LaTeX font definitions
+)
+Package microtype Info: Loading generic protrusion settings for font family
+(microtype) `cmtt' (encoding: T1).
+(microtype) For optimal results, create family-specific settings.
+(microtype) See the microtype manual for details.
+LaTeX Font Info: Trying to load font information for T1+phv on input line 93.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/t1phv.fd
+File: t1phv.fd 2020/03/25 scalable font definitions for T1/phv.
+)
+Package microtype Info: Loading generic protrusion settings for font family
+(microtype) `phv' (encoding: T1).
+(microtype) For optimal results, create family-specific settings.
+(microtype) See the microtype manual for details.
+ [1
+
+
+{/usr/local/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map}{/usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/base/8r.enc}{/usr/local/texlive/2023/texmf-dist/fonts/enc/dvips/cm-super/cm-super-t1.enc}] [2] [3]
+LaTeX Font Info: Trying to load font information for TS1+ptm on input line 382.
+ (/usr/local/texlive/2023/texmf-dist/tex/latex/psnfss/ts1ptm.fd
+File: ts1ptm.fd 2001/06/04 font definitions for TS1/ptm.
+) [4] [5] [6] (./paper.aux)
+Package rerunfilecheck Info: File `paper.out' has not changed.
+(rerunfilecheck) Checksum: ADC4F3226FC4BAD2B7E73B5F36623AD6;2443.
+ )
+Here is how much of TeX's memory you used:
+ 12669 strings out of 476025
+ 200194 string characters out of 5790017
+ 1868388 words of memory out of 5000000
+ 32677 multiletter control sequences out of 15000+600000
+ 563302 words of font info for 205 fonts, out of 8000000 for 9000
+ 1141 hyphenation exceptions out of 8191
+ 75i,9n,76p,1074b,651s stack positions out of 10000i,1000n,20000p,200000b,200000s
+</usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr7.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/symbols/msbm10.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/public/cm-super/sftt1000.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/urw/helvetic/uhvr8a.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmb8a.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmr8a.pfb></usr/local/texlive/2023/texmf-dist/fonts/type1/urw/times/utmri8a.pfb>
+Output written on paper.pdf (6 pages, 150673 bytes).
+PDF statistics:
+ 211 PDF objects out of 1000 (max. 8388607)
+ 181 compressed objects within 2 object streams
+ 54 named destinations out of 1000 (max. 500000)
+ 58497 words of extra memory for PDF output out of 61914 (max. 10000000)
+
diff --git a/paper/paper.out b/paper/paper.out
new file mode 100644
index 0000000000000000000000000000000000000000..c20c9d2517ae00959b3451dcd79dd441aa9ebcb9
--- /dev/null
+++ b/paper/paper.out
@@ -0,0 +1,16 @@
+\BOOKMARK [1][-]{section.1}{\376\377\000S\000u\000b\000m\000i\000s\000s\000i\000o\000n\000\040\000o\000f\000\040\000p\000a\000p\000e\000r\000s\000\040\000t\000o\000\040\000N\000e\000u\000r\000I\000P\000S\000\040\0002\0000\0002\0002}{}% 1
+\BOOKMARK [2][-]{subsection.1.1}{\376\377\000S\000t\000y\000l\000e}{section.1}% 2
+\BOOKMARK [2][-]{subsection.1.2}{\376\377\000R\000e\000t\000r\000i\000e\000v\000a\000l\000\040\000o\000f\000\040\000s\000t\000y\000l\000e\000\040\000f\000i\000l\000e\000s}{section.1}% 3
+\BOOKMARK [1][-]{section.2}{\376\377\000G\000e\000n\000e\000r\000a\000l\000\040\000f\000o\000r\000m\000a\000t\000t\000i\000n\000g\000\040\000i\000n\000s\000t\000r\000u\000c\000t\000i\000o\000n\000s}{}% 4
+\BOOKMARK [1][-]{section.3}{\376\377\000H\000e\000a\000d\000i\000n\000g\000s\000:\000\040\000f\000i\000r\000s\000t\000\040\000l\000e\000v\000e\000l}{}% 5
+\BOOKMARK [2][-]{subsection.3.1}{\376\377\000H\000e\000a\000d\000i\000n\000g\000s\000:\000\040\000s\000e\000c\000o\000n\000d\000\040\000l\000e\000v\000e\000l}{section.3}% 6
+\BOOKMARK [3][-]{subsubsection.3.1.1}{\376\377\000H\000e\000a\000d\000i\000n\000g\000s\000:\000\040\000t\000h\000i\000r\000d\000\040\000l\000e\000v\000e\000l}{subsection.3.1}% 7
+\BOOKMARK [1][-]{section.4}{\376\377\000C\000i\000t\000a\000t\000i\000o\000n\000s\000,\000\040\000f\000i\000g\000u\000r\000e\000s\000,\000\040\000t\000a\000b\000l\000e\000s\000,\000\040\000r\000e\000f\000e\000r\000e\000n\000c\000e\000s}{}% 8
+\BOOKMARK [2][-]{subsection.4.1}{\376\377\000C\000i\000t\000a\000t\000i\000o\000n\000s\000\040\000w\000i\000t\000h\000i\000n\000\040\000t\000h\000e\000\040\000t\000e\000x\000t}{section.4}% 9
+\BOOKMARK [2][-]{subsection.4.2}{\376\377\000F\000o\000o\000t\000n\000o\000t\000e\000s}{section.4}% 10
+\BOOKMARK [2][-]{subsection.4.3}{\376\377\000F\000i\000g\000u\000r\000e\000s}{section.4}% 11
+\BOOKMARK [2][-]{subsection.4.4}{\376\377\000T\000a\000b\000l\000e\000s}{section.4}% 12
+\BOOKMARK [1][-]{section.5}{\376\377\000F\000i\000n\000a\000l\000\040\000i\000n\000s\000t\000r\000u\000c\000t\000i\000o\000n\000s}{}% 13
+\BOOKMARK [1][-]{section.6}{\376\377\000P\000r\000e\000p\000a\000r\000i\000n\000g\000\040\000P\000D\000F\000\040\000f\000i\000l\000e\000s}{}% 14
+\BOOKMARK [2][-]{subsection.6.1}{\376\377\000M\000a\000r\000g\000i\000n\000s\000\040\000i\000n\000\040\000L\000a\000T\000e\000X}{section.6}% 15
+\BOOKMARK [1][-]{appendix.A}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x}{}% 16
diff --git a/paper/paper.pdf b/paper/paper.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..42b451df30e63ec2c98399e7e104f36fbdcde099
Binary files /dev/null and b/paper/paper.pdf differ
diff --git a/paper/paper.synctex.gz b/paper/paper.synctex.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b4749b3e1a488384c8aab85331901ad956cc4abc
Binary files /dev/null and b/paper/paper.synctex.gz differ
diff --git a/paper/paper.tex b/paper/paper.tex
new file mode 100644
index 0000000000000000000000000000000000000000..585f6921708190c0740766c6069d16d03e56d6ab
--- /dev/null
+++ b/paper/paper.tex
@@ -0,0 +1,589 @@
+\documentclass{article}
+
+
+% if you need to pass options to natbib, use, e.g.:
+% \PassOptionsToPackage{numbers, compress}{natbib}
+% before loading neurips_2022
+
+
+% ready for submission
+\usepackage{neurips_2022}
+
+
+% to compile a preprint version, e.g., for submission to arXiv, add add the
+% [preprint] option:
+% \usepackage[preprint]{neurips_2022}
+
+
+% to compile a camera-ready version, add the [final] option, e.g.:
+% \usepackage[final]{neurips_2022}
+
+
+% to avoid loading the natbib package, add option nonatbib:
+% \usepackage[nonatbib]{neurips_2022}
+
+
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc} % use 8-bit T1 fonts
+\usepackage{hyperref} % hyperlinks
+\usepackage{url} % simple URL typesetting
+\usepackage{booktabs} % professional-quality tables
+\usepackage{amsfonts} % blackboard math symbols
+\usepackage{nicefrac} % compact symbols for 1/2, etc.
+\usepackage{microtype} % microtypography
+\usepackage{xcolor} % colors
+
+
+\title{Formatting Instructions For NeurIPS 2022}
+
+
+% The \author macro works with any number of authors. There are two commands
+% used to separate the names and addresses of multiple authors: \And and \AND.
+%
+% Using \And between authors leaves it to LaTeX to determine where to break the
+% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
+% authors names on the first line, and the last on the second line, try using
+% \AND instead of \And before the third author name.
+
+
+\author{%
+ David S.~Hippocampus\thanks{Use footnote for providing further information
+ about author (webpage, alternative address)---\emph{not} for acknowledging
+ funding agencies.} \\
+ Department of Computer Science\\
+ Cranberry-Lemon University\\
+ Pittsburgh, PA 15213 \\
+ \texttt{hippo@cs.cranberry-lemon.edu} \\
+ % examples of more authors
+ % \And
+ % Coauthor \\
+ % Affiliation \\
+ % Address \\
+ % \texttt{email} \\
+ % \AND
+ % Coauthor \\
+ % Affiliation \\
+ % Address \\
+ % \texttt{email} \\
+ % \And
+ % Coauthor \\
+ % Affiliation \\
+ % Address \\
+ % \texttt{email} \\
+ % \And
+ % Coauthor \\
+ % Affiliation \\
+ % Address \\
+ % \texttt{email} \\
+}
+
+
+\begin{document}
+
+
+\maketitle
+
+
+\begin{abstract}
+ The abstract paragraph should be indented \nicefrac{1}{2}~inch (3~picas) on
+ both the left- and right-hand margins. Use 10~point type, with a vertical
+ spacing (leading) of 11~points. The word \textbf{Abstract} must be centered,
+ bold, and in point size 12. Two line spaces precede the abstract. The abstract
+ must be limited to one paragraph.
+\end{abstract}
+
+
+\section{Submission of papers to NeurIPS 2022}
+
+
+Please read the instructions below carefully and follow them faithfully.
+
+
+\subsection{Style}
+
+
+Papers to be submitted to NeurIPS 2022 must be prepared according to the
+instructions presented here. Papers may only be up to {\bf nine} pages long,
+including figures. Additional pages \emph{containing only acknowledgments and
+references} are allowed. Papers that exceed the page limit will not be
+reviewed, or in any other way considered for presentation at the conference.
+
+
+The margins in 2022 are the same as those in 2007, which allow for $\sim$$15\%$
+more words in the paper compared to earlier years.
+
+
+Authors are required to use the NeurIPS \LaTeX{} style files obtainable at the
+NeurIPS website as indicated below. Please make sure you use the current files
+and not previous versions. Tweaking the style files may be grounds for
+rejection.
+
+
+\subsection{Retrieval of style files}
+
+
+The style files for NeurIPS and other conference information are available on
+the World Wide Web at
+\begin{center}
+ \url{http://www.neurips.cc/}
+\end{center}
+The file \verb+neurips_2022.pdf+ contains these instructions and illustrates the
+various formatting requirements your NeurIPS paper must satisfy.
+
+
+The only supported style file for NeurIPS 2022 is \verb+neurips_2022.sty+,
+rewritten for \LaTeXe{}. \textbf{Previous style files for \LaTeX{} 2.09,
+ Microsoft Word, and RTF are no longer supported!}
+
+
+The \LaTeX{} style file contains three optional arguments: \verb+final+, which
+creates a camera-ready copy, \verb+preprint+, which creates a preprint for
+submission to, e.g., arXiv, and \verb+nonatbib+, which will not load the
+\verb+natbib+ package for you in case of package clash.
+
+
+\paragraph{Preprint option}
+If you wish to post a preprint of your work online, e.g., on arXiv, using the
+NeurIPS style, please use the \verb+preprint+ option. This will create a
+nonanonymized version of your work with the text ``Preprint. Work in progress.''
+in the footer. This version may be distributed as you see fit. Please \textbf{do
+ not} use the \verb+final+ option, which should \textbf{only} be used for
+papers accepted to NeurIPS.
+
+
+At submission time, please omit the \verb+final+ and \verb+preprint+
+options. This will anonymize your submission and add line numbers to aid
+review. Please do \emph{not} refer to these line numbers in your paper as they
+will be removed during generation of camera-ready copies.
+
+
+The file \verb+neurips_2022.tex+ may be used as a ``shell'' for writing your
+paper. All you have to do is replace the author, title, abstract, and text of
+the paper with your own.
+
+
+The formatting instructions contained in these style files are summarized in
+Sections \ref{gen_inst}, \ref{headings}, and \ref{others} below.
+
+
+\section{General formatting instructions}
+\label{gen_inst}
+
+
+The text must be confined within a rectangle 5.5~inches (33~picas) wide and
+9~inches (54~picas) long. The left margin is 1.5~inch (9~picas). Use 10~point
+type with a vertical spacing (leading) of 11~points. Times New Roman is the
+preferred typeface throughout, and will be selected for you by default.
+Paragraphs are separated by \nicefrac{1}{2}~line space (5.5 points), with no
+indentation.
+
+
+The paper title should be 17~point, initial caps/lower case, bold, centered
+between two horizontal rules. The top rule should be 4~points thick and the
+bottom rule should be 1~point thick. Allow \nicefrac{1}{4}~inch space above and
+below the title to rules. All pages should start at 1~inch (6~picas) from the
+top of the page.
+
+
+For the final version, authors' names are set in boldface, and each name is
+centered above the corresponding address. The lead author's name is to be listed
+first (left-most), and the co-authors' names (if different address) are set to
+follow. If there is only one co-author, list both author and co-author side by
+side.
+
+
+Please pay special attention to the instructions in Section \ref{others}
+regarding figures, tables, acknowledgments, and references.
+
+
+\section{Headings: first level}
+\label{headings}
+
+
+All headings should be lower case (except for first word and proper nouns),
+flush left, and bold.
+
+
+First-level headings should be in 12-point type.
+
+
+\subsection{Headings: second level}
+
+
+Second-level headings should be in 10-point type.
+
+
+\subsubsection{Headings: third level}
+
+
+Third-level headings should be in 10-point type.
+
+
+\paragraph{Paragraphs}
+
+
+There is also a \verb+\paragraph+ command available, which sets the heading in
+bold, flush left, and inline with the text, with the heading followed by 1\,em
+of space.
+
+
+\section{Citations, figures, tables, references}
+\label{others}
+
+
+These instructions apply to everyone.
+
+
+\subsection{Citations within the text}
+
+
+The \verb+natbib+ package will be loaded for you by default. Citations may be
+author/year or numeric, as long as you maintain internal consistency. As to the
+format of the references themselves, any style is acceptable as long as it is
+used consistently.
+
+
+The documentation for \verb+natbib+ may be found at
+\begin{center}
+ \url{http://mirrors.ctan.org/macros/latex/contrib/natbib/natnotes.pdf}
+\end{center}
+Of note is the command \verb+\citet+, which produces citations appropriate for
+use in inline text. For example,
+\begin{verbatim}
+ \citet{hasselmo} investigated\dots
+\end{verbatim}
+produces
+\begin{quote}
+ Hasselmo, et al.\ (1995) investigated\dots
+\end{quote}
+
+
+If you wish to load the \verb+natbib+ package with options, you may add the
+following before loading the \verb+neurips_2022+ package:
+\begin{verbatim}
+ \PassOptionsToPackage{options}{natbib}
+\end{verbatim}
+
+
+If \verb+natbib+ clashes with another package you load, you can add the optional
+argument \verb+nonatbib+ when loading the style file:
+\begin{verbatim}
+ \usepackage[nonatbib]{neurips_2022}
+\end{verbatim}
+
+
+As submission is double blind, refer to your own published work in the third
+person. That is, use ``In the previous work of Jones et al.\ [4],'' not ``In our
+previous work [4].'' If you cite your other papers that are not widely available
+(e.g., a journal paper under review), use anonymous author names in the
+citation, e.g., an author of the form ``A.\ Anonymous.''
+
+
+\subsection{Footnotes}
+
+
+Footnotes should be used sparingly. If you do require a footnote, indicate
+footnotes with a number\footnote{Sample of the first footnote.} in the
+text. Place the footnotes at the bottom of the page on which they appear.
+Precede the footnote with a horizontal rule of 2~inches (12~picas).
+
+
+Note that footnotes are properly typeset \emph{after} punctuation
+marks.\footnote{As in this example.}
+
+
+\subsection{Figures}
+
+
+\begin{figure}
+ \centering
+ \fbox{\rule[-.5cm]{0cm}{4cm} \rule[-.5cm]{4cm}{0cm}}
+ \caption{Sample figure caption.}
+\end{figure}
+
+
+All artwork must be neat, clean, and legible. Lines should be dark enough for
+purposes of reproduction. The figure number and caption always appear after the
+figure. Place one line space before the figure caption and one line space after
+the figure. The figure caption should be lower case (except for first word and
+proper nouns); figures are numbered consecutively.
+
+
+You may use color figures. However, it is best for the figure captions and the
+paper body to be legible if the paper is printed in either black/white or in
+color.
+
+
+\subsection{Tables}
+
+
+All tables must be centered, neat, clean and legible. The table number and
+title always appear before the table. See Table~\ref{sample-table}.
+
+
+Place one line space before the table title, one line space after the
+table title, and one line space after the table. The table title must
+be lower case (except for first word and proper nouns); tables are
+numbered consecutively.
+
+
+Note that publication-quality tables \emph{do not contain vertical rules.} We
+strongly suggest the use of the \verb+booktabs+ package, which allows for
+typesetting high-quality, professional tables:
+\begin{center}
+ \url{https://www.ctan.org/pkg/booktabs}
+\end{center}
+This package was used to typeset Table~\ref{sample-table}.
+
+
+\begin{table}
+ \caption{Sample table title}
+ \label{sample-table}
+ \centering
+ \begin{tabular}{lll}
+ \toprule
+ \multicolumn{2}{c}{Part} \\
+ \cmidrule(r){1-2}
+ Name & Description & Size ($\mu$m) \\
+ \midrule
+ Dendrite & Input terminal & $\sim$100 \\
+ Axon & Output terminal & $\sim$10 \\
+ Soma & Cell body & up to $10^6$ \\
+ \bottomrule
+ \end{tabular}
+\end{table}
+
+
+\section{Final instructions}
+
+
+Do not change any aspects of the formatting parameters in the style files. In
+particular, do not modify the width or length of the rectangle the text should
+fit into, and do not change font sizes (except perhaps in the
+\textbf{References} section; see below). Please note that pages should be
+numbered.
+
+
+\section{Preparing PDF files}
+
+
+Please prepare submission files with paper size ``US Letter,'' and not, for
+example, ``A4.''
+
+
+Fonts were the main cause of problems in the past years. Your PDF file must only
+contain Type 1 or Embedded TrueType fonts. Here are a few instructions to
+achieve this.
+
+
+\begin{itemize}
+
+
+\item You should directly generate PDF files using \verb+pdflatex+.
+
+
+\item You can check which fonts a PDF files uses. In Acrobat Reader, select the
+ menu Files$>$Document Properties$>$Fonts and select Show All Fonts. You can
+ also use the program \verb+pdffonts+ which comes with \verb+xpdf+ and is
+ available out-of-the-box on most Linux machines.
+
+
+\item The IEEE has recommendations for generating PDF files whose fonts are also
+ acceptable for NeurIPS. Please see
+ \url{http://www.emfield.org/icuwb2010/downloads/IEEE-PDF-SpecV32.pdf}
+
+
+\item \verb+xfig+ "patterned" shapes are implemented with bitmap fonts. Use
+ "solid" shapes instead.
+
+
+\item The \verb+\bbold+ package almost always uses bitmap fonts. You should use
+ the equivalent AMS Fonts:
+\begin{verbatim}
+ \usepackage{amsfonts}
+\end{verbatim}
+followed by, e.g., \verb+\mathbb{R}+, \verb+\mathbb{N}+, or \verb+\mathbb{C}+
+for $\mathbb{R}$, $\mathbb{N}$ or $\mathbb{C}$. You can also use the following
+workaround for reals, natural and complex:
+\begin{verbatim}
+ \newcommand{\RR}{I\!\!R} %real numbers
+ \newcommand{\Nat}{I\!\!N} %natural numbers
+ \newcommand{\CC}{I\!\!\!\!C} %complex numbers
+\end{verbatim}
+Note that \verb+amsfonts+ is automatically loaded by the \verb+amssymb+ package.
+
+
+\end{itemize}
+
+
+If your file contains type 3 fonts or non embedded TrueType fonts, we will ask
+you to fix it.
+
+
+\subsection{Margins in \LaTeX{}}
+
+
+Most of the margin problems come from figures positioned by hand using
+\verb+\special+ or other commands. We suggest using the command
+\verb+\includegraphics+ from the \verb+graphicx+ package. Always specify the
+figure width as a multiple of the line width as in the example below:
+\begin{verbatim}
+ \usepackage[pdftex]{graphicx} ...
+ \includegraphics[width=0.8\linewidth]{myfile.pdf}
+\end{verbatim}
+See Section 4.4 in the graphics bundle documentation
+(\url{http://mirrors.ctan.org/macros/latex/required/graphics/grfguide.pdf})
+
+
+A number of width problems arise when \LaTeX{} cannot properly hyphenate a
+line. Please give LaTeX hyphenation hints using the \verb+\-+ command when
+necessary.
+
+
+\begin{ack}
+Use unnumbered first level headings for the acknowledgments. All acknowledgments
+go at the end of the paper before the list of references. Moreover, you are required to declare
+funding (financial activities supporting the submitted work) and competing interests (related financial activities outside the submitted work).
+More information about this disclosure can be found at: \url{https://neurips.cc/Conferences/2022/PaperInformation/FundingDisclosure}.
+
+
+Do {\bf not} include this section in the anonymized submission, only in the final paper. You can use the \texttt{ack} environment provided in the style file to autmoatically hide this section in the anonymized submission.
+\end{ack}
+
+
+\section*{References}
+
+
+References follow the acknowledgments. Use unnumbered first-level heading for
+the references. Any choice of citation style is acceptable as long as you are
+consistent. It is permissible to reduce the font size to \verb+small+ (9 point)
+when listing the references.
+Note that the Reference section does not count towards the page limit.
+\medskip
+
+
+{
+\small
+
+
+[1] Alexander, J.A.\ \& Mozer, M.C.\ (1995) Template-based algorithms for
+connectionist rule extraction. In G.\ Tesauro, D.S.\ Touretzky and T.K.\ Leen
+(eds.), {\it Advances in Neural Information Processing Systems 7},
+pp.\ 609--616. Cambridge, MA: MIT Press.
+
+
+[2] Bower, J.M.\ \& Beeman, D.\ (1995) {\it The Book of GENESIS: Exploring
+ Realistic Neural Models with the GEneral NEural SImulation System.} New York:
+TELOS/Springer--Verlag.
+
+
+[3] Hasselmo, M.E., Schnell, E.\ \& Barkai, E.\ (1995) Dynamics of learning and
+recall at excitatory recurrent synapses and cholinergic modulation in rat
+hippocampal region CA3. {\it Journal of Neuroscience} {\bf 15}(7):5249-5262.
+}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\section*{Checklist}
+
+
+%%% BEGIN INSTRUCTIONS %%%
+The checklist follows the references. Please
+read the checklist guidelines carefully for information on how to answer these
+questions. For each question, change the default \answerTODO{} to \answerYes{},
+\answerNo{}, or \answerNA{}. You are strongly encouraged to include a {\bf
+justification to your answer}, either by referencing the appropriate section of
+your paper or providing a brief inline description. For example:
+\begin{itemize}
+ \item Did you include the license to the code and datasets? \answerYes{See Section~\ref{gen_inst}.}
+ \item Did you include the license to the code and datasets? \answerNo{The code and the data are proprietary.}
+ \item Did you include the license to the code and datasets? \answerNA{}
+\end{itemize}
+Please do not modify the questions and only use the provided macros for your
+answers. Note that the Checklist section does not count towards the page
+limit. In your paper, please delete this instructions block and only keep the
+Checklist section heading above along with the questions/answers below.
+%%% END INSTRUCTIONS %%%
+
+
+\begin{enumerate}
+
+
+\item For all authors...
+\begin{enumerate}
+ \item Do the main claims made in the abstract and introduction accurately reflect the paper's contributions and scope?
+ \answerTODO{}
+ \item Did you describe the limitations of your work?
+ \answerTODO{}
+ \item Did you discuss any potential negative societal impacts of your work?
+ \answerTODO{}
+ \item Have you read the ethics review guidelines and ensured that your paper conforms to them?
+ \answerTODO{}
+\end{enumerate}
+
+
+\item If you are including theoretical results...
+\begin{enumerate}
+ \item Did you state the full set of assumptions of all theoretical results?
+ \answerTODO{}
+ \item Did you include complete proofs of all theoretical results?
+ \answerTODO{}
+\end{enumerate}
+
+
+\item If you ran experiments...
+\begin{enumerate}
+ \item Did you include the code, data, and instructions needed to reproduce the main experimental results (either in the supplemental material or as a URL)?
+ \answerTODO{}
+ \item Did you specify all the training details (e.g., data splits, hyperparameters, how they were chosen)?
+ \answerTODO{}
+ \item Did you report error bars (e.g., with respect to the random seed after running experiments multiple times)?
+ \answerTODO{}
+ \item Did you include the total amount of compute and the type of resources used (e.g., type of GPUs, internal cluster, or cloud provider)?
+ \answerTODO{}
+\end{enumerate}
+
+
+\item If you are using existing assets (e.g., code, data, models) or curating/releasing new assets...
+\begin{enumerate}
+ \item If your work uses existing assets, did you cite the creators?
+ \answerTODO{}
+ \item Did you mention the license of the assets?
+ \answerTODO{}
+ \item Did you include any new assets either in the supplemental material or as a URL?
+ \answerTODO{}
+ \item Did you discuss whether and how consent was obtained from people whose data you're using/curating?
+ \answerTODO{}
+ \item Did you discuss whether the data you are using/curating contains personally identifiable information or offensive content?
+ \answerTODO{}
+\end{enumerate}
+
+
+\item If you used crowdsourcing or conducted research with human subjects...
+\begin{enumerate}
+ \item Did you include the full text of instructions given to participants and screenshots, if applicable?
+ \answerTODO{}
+ \item Did you describe any potential participant risks, with links to Institutional Review Board (IRB) approvals, if applicable?
+ \answerTODO{}
+ \item Did you include the estimated hourly wage paid to participants and the total amount spent on participant compensation?
+ \answerTODO{}
+\end{enumerate}
+
+
+\end{enumerate}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+\appendix
+
+
+\section{Appendix}
+
+
+Optionally include extra information (complete proofs, additional experiments and plots) in the appendix.
+This section will often be part of the supplemental material.
+
+
+\end{document}
\ No newline at end of file