diff --git a/artifacts/results/gmsc_benchmark.csv b/artifacts/results/gmsc_benchmark.csv new file mode 100644 index 0000000000000000000000000000000000000000..9478a257a70d6b3d02b172e9c19d4e14640748a8 --- /dev/null +++ b/artifacts/results/gmsc_benchmark.csv @@ -0,0 +1,801 @@ +sample,num_counterfactual,variable,value,ce,dataname,generator,model,target,factual +1,1,distance,Float32[1.8808396]," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +1,1,distance_from_energy,8.461664," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +1,1,distance_from_targets,4.557821," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +1,1,redundancy,0.0," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +1,1,validity,false," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +2,1,distance,Float32[3.8812537]," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +2,1,distance_from_energy,10.366544," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +2,1,distance_from_targets,4.504924," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +2,1,redundancy,0.0," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +2,1,validity,false," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +3,1,distance,Float32[1.8619741]," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +3,1,distance_from_energy,10.0912," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +3,1,distance_from_targets,5.72349," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +3,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +3,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +4,1,distance,Float32[1.8302257]," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +4,1,distance_from_energy,9.54855," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +4,1,distance_from_targets,5.3711357," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +4,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +4,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM Ensemble,1,0 +5,1,distance,Float32[2.374693]," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +5,1,distance_from_energy,10.37821," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +5,1,distance_from_targets,6.7792215," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +5,1,redundancy,0.0," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +5,1,validity,false," +Convergence: âŒ",Californian Housing,revise,JEM Ensemble,1,0 +1,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +1,1,distance_from_energy,10.128708," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +1,1,distance_from_targets,5.6870294," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +1,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +1,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +2,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +2,1,distance_from_energy,8.044174," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +2,1,distance_from_targets,6.587344," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +2,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +2,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +3,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +3,1,distance_from_energy,10.562269," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +3,1,distance_from_targets,6.5746174," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +3,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +3,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +4,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +4,1,distance_from_energy,10.433871," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +4,1,distance_from_targets,6.1688733," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +4,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +4,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +5,1,distance,Float32[0.70000017]," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +5,1,distance_from_energy,9.828283," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +5,1,distance_from_targets,7.0533257," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +5,1,redundancy,0.75," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +5,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM Ensemble,1,0 +1,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +1,1,distance_from_energy,9.726099," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +1,1,distance_from_targets,5.4704204," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +1,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +1,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +2,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +2,1,distance_from_energy,9.422539," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +2,1,distance_from_targets,6.3577952," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +2,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +2,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +3,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +3,1,distance_from_energy,9.744759," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +3,1,distance_from_targets,6.589761," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +3,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +3,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +4,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +4,1,distance_from_energy,9.709781," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +4,1,distance_from_targets,6.180225," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +4,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +4,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +5,1,distance,Float32[0.8000001]," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +5,1,distance_from_energy,10.758295," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +5,1,distance_from_targets,6.8113194," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +5,1,redundancy,0.75," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +5,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM Ensemble,1,0 +1,1,distance,Float32[0.6799995]," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +1,1,distance_from_energy,8.298499," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +1,1,distance_from_targets,5.641716," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +1,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +1,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +2,1,distance,Float32[0.31999975]," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +2,1,distance_from_energy,7.664296," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +2,1,distance_from_targets,6.6763825," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +2,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +2,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +3,1,distance,Float32[0.07999997]," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +3,1,distance_from_energy,10.768106," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +3,1,distance_from_targets,6.361887," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +3,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +3,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +4,1,distance,Float32[0.35999972]," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +4,1,distance_from_energy,8.668391," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +4,1,distance_from_targets,6.2152205," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +4,1,redundancy,0.25," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +4,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +5,1,distance,Float32[0.83999944]," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +5,1,distance_from_energy,8.757013," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +5,1,distance_from_targets,6.563821," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +5,1,redundancy,0.25," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +5,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM Ensemble,1,0 +6,1,distance,Float32[4.0026846]," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +6,1,distance_from_energy,314.77728," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +6,1,distance_from_targets,5.6145387," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +6,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +6,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +7,1,distance,Float32[3.2443917]," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +7,1,distance_from_energy,322.58966," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +7,1,distance_from_targets,4.734803," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +7,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +7,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +8,1,distance,Float32[4.6039753]," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +8,1,distance_from_energy,406.99112," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +8,1,distance_from_targets,4.7767906," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +8,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +8,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +9,1,distance,Float32[1.6108489]," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +9,1,distance_from_energy,259.97772," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +9,1,distance_from_targets,4.821418," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +9,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +9,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +10,1,distance,Float32[3.361702]," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +10,1,distance_from_energy,325.50375," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +10,1,distance_from_targets,5.769852," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +10,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +10,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP,1,0 +6,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +6,1,distance_from_energy,177.16382," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +6,1,distance_from_targets,7.6693," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +6,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +6,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +7,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +7,1,distance_from_energy,170.38297," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +7,1,distance_from_targets,6.680264," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +7,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +7,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +8,1,distance,Float32[0.4]," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +8,1,distance_from_energy,216.61328," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +8,1,distance_from_targets,6.4001665," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +8,1,redundancy,0.75," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +8,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +9,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +9,1,distance_from_energy,281.91168," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +9,1,distance_from_targets,5.159766," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +9,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +9,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +10,1,distance,Float32[0.59999996]," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +10,1,distance_from_energy,219.05847," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +10,1,distance_from_targets,7.6656485," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +10,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +10,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP,1,0 +6,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +6,1,distance_from_energy,246.38237," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +6,1,distance_from_targets,7.6779747," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +6,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +6,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +7,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +7,1,distance_from_energy,297.0213," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +7,1,distance_from_targets,6.6958055," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +7,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +7,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +8,1,distance,Float32[0.4]," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +8,1,distance_from_energy,266.59482," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +8,1,distance_from_targets,6.331769," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +8,1,redundancy,0.75," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +8,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +9,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +9,1,distance_from_energy,290.03915," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +9,1,distance_from_targets,5.263832," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +9,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +9,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +10,1,distance,Float32[0.59999996]," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +10,1,distance_from_energy,301.52667," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +10,1,distance_from_targets,7.647933," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +10,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +10,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP,1,0 +6,1,distance,Float32[0.95999956]," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +6,1,distance_from_energy,246.16826," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +6,1,distance_from_targets,7.4335694," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +6,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +6,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +7,1,distance,Float32[0.35999975]," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +7,1,distance_from_energy,228.79973," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +7,1,distance_from_targets,6.351187," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +7,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +7,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +8,1,distance,Float32[0.79999954]," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +8,1,distance_from_energy,218.44572," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +8,1,distance_from_targets,6.1676702," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +8,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +8,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +9,1,distance,Float32[0.07999994]," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +9,1,distance_from_energy,196.818," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +9,1,distance_from_targets,5.023533," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +9,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +9,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +10,1,distance,Float32[1.6399989]," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +10,1,distance_from_energy,268.28333," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +10,1,distance_from_targets,7.4489574," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +10,1,redundancy,0.125," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +10,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP,1,0 +11,1,distance,Float32[2.988908]," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +11,1,distance_from_energy,166.39853," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +11,1,distance_from_targets,5.394507," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +11,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +11,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +12,1,distance,Float32[8.518592]," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +12,1,distance_from_energy,197.99025," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +12,1,distance_from_targets,4.6680164," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +12,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +12,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +13,1,distance,Float32[1.5909717]," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +13,1,distance_from_energy,246.94055," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +13,1,distance_from_targets,4.8824925," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +13,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +13,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +14,1,distance,Float32[3.789693]," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +14,1,distance_from_energy,187.41772," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +14,1,distance_from_targets,4.701274," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +14,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +14,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +15,1,distance,Float32[3.7671118]," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +15,1,distance_from_energy,308.73193," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +15,1,distance_from_targets,4.6771507," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +15,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +15,1,validity,true," +Convergence: ✅",Californian Housing,revise,MLP Ensemble,1,0 +11,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +11,1,distance_from_energy,214.13828," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +11,1,distance_from_targets,7.074875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +11,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +11,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +12,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +12,1,distance_from_energy,214.93787," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +12,1,distance_from_targets,8.603035," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +12,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +12,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +13,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +13,1,distance_from_energy,262.8469," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +13,1,distance_from_targets,5.794109," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +13,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +13,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +14,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +14,1,distance_from_energy,289.6086," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +14,1,distance_from_targets,6.75391," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +14,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +14,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +15,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +15,1,distance_from_energy,219.26624," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +15,1,distance_from_targets,6.5528407," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +15,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +15,1,validity,true," +Convergence: ✅",Californian Housing,greedy,MLP Ensemble,1,0 +11,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +11,1,distance_from_energy,306.83655," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +11,1,distance_from_targets,7.1254005," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +11,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +11,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +12,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +12,1,distance_from_energy,264.65982," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +12,1,distance_from_targets,8.556416," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +12,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +12,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +13,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +13,1,distance_from_energy,210.39725," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +13,1,distance_from_targets,5.6687," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +13,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +13,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +14,1,distance,Float32[0.39999998]," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +14,1,distance_from_energy,299.83313," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +14,1,distance_from_targets,6.7239194," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +14,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +14,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +15,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +15,1,distance_from_energy,283.7801," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +15,1,distance_from_targets,6.476427," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +15,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +15,1,validity,true," +Convergence: ✅",Californian Housing,eccco,MLP Ensemble,1,0 +11,1,distance,Float32[0.87999946]," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +11,1,distance_from_energy,240.0104," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +11,1,distance_from_targets,6.619682," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +11,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +11,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +12,1,distance,Float32[0.13999996]," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +12,1,distance_from_energy,210.00273," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +12,1,distance_from_targets,8.638436," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +12,1,redundancy,0.125," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +12,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +13,1,distance,Float32[0.07999995]," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +13,1,distance_from_energy,338.97745," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +13,1,distance_from_targets,5.7860813," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +13,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +13,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +14,1,distance,Float32[1.3199995]," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +14,1,distance_from_energy,254.07788," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +14,1,distance_from_targets,6.563486," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +14,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +14,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +15,1,distance,Float32[0.1399999]," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +15,1,distance_from_energy,222.0273," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +15,1,distance_from_targets,6.601179," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +15,1,redundancy,0.125," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +15,1,validity,true," +Convergence: ✅",Californian Housing,wachter,MLP Ensemble,1,0 +16,1,distance,Float32[2.9892344]," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +16,1,distance_from_energy,8.488709," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +16,1,distance_from_targets,4.4637537," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +16,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +16,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +17,1,distance,Float32[2.5778935]," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +17,1,distance_from_energy,8.316306," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +17,1,distance_from_targets,4.7104325," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +17,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +17,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +18,1,distance,Float32[3.0227609]," +Convergence: âŒ",Californian Housing,revise,JEM,1,0 +18,1,distance_from_energy,9.328789," +Convergence: âŒ",Californian Housing,revise,JEM,1,0 +18,1,distance_from_targets,4.8747354," +Convergence: âŒ",Californian Housing,revise,JEM,1,0 +18,1,redundancy,0.0," +Convergence: âŒ",Californian Housing,revise,JEM,1,0 +18,1,validity,false," +Convergence: âŒ",Californian Housing,revise,JEM,1,0 +19,1,distance,Float32[2.872929]," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +19,1,distance_from_energy,10.106264," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +19,1,distance_from_targets,5.788596," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +19,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +19,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +20,1,distance,Float32[2.173339]," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +20,1,distance_from_energy,8.9135," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +20,1,distance_from_targets,5.000705," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +20,1,redundancy,0.0," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +20,1,validity,true," +Convergence: ✅",Californian Housing,revise,JEM,1,0 +16,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +16,1,distance_from_energy,9.9056835," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +16,1,distance_from_targets,6.3118243," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +16,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +16,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +17,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +17,1,distance_from_energy,8.909609," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +17,1,distance_from_targets,5.394095," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +17,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +17,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +18,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +18,1,distance_from_energy,10.332393," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +18,1,distance_from_targets,6.551979," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +18,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +18,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +19,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +19,1,distance_from_energy,9.619695," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +19,1,distance_from_targets,7.0109854," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +19,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +19,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +20,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +20,1,distance_from_energy,9.4669285," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +20,1,distance_from_targets,5.6579533," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +20,1,redundancy,0.875," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +20,1,validity,true," +Convergence: ✅",Californian Housing,greedy,JEM,1,0 +16,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +16,1,distance_from_energy,11.167322," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +16,1,distance_from_targets,6.1822476," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +16,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +16,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +17,1,distance,Float32[0.099999994]," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +17,1,distance_from_energy,9.06465," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +17,1,distance_from_targets,5.4826946," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +17,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +17,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +18,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +18,1,distance_from_energy,9.936276," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +18,1,distance_from_targets,6.5694094," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +18,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +18,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +19,1,distance,Float32[0.1]," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +19,1,distance_from_energy,10.191511," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +19,1,distance_from_targets,6.9682," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +19,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +19,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +20,1,distance,Float32[0.2]," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +20,1,distance_from_energy,10.2408695," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +20,1,distance_from_targets,5.6725516," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +20,1,redundancy,0.875," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +20,1,validity,true," +Convergence: ✅",Californian Housing,eccco,JEM,1,0 +16,1,distance,Float32[0.43999973]," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +16,1,distance_from_energy,10.513639," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +16,1,distance_from_targets,6.377578," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +16,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +16,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +17,1,distance,Float32[0.2799998]," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +17,1,distance_from_energy,6.898867," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +17,1,distance_from_targets,5.364885," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +17,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +17,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +18,1,distance,Float32[0.6799996]," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +18,1,distance_from_energy,9.212458," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +18,1,distance_from_targets,6.529666," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +18,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +18,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +19,1,distance,Float32[0.4999996]," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +19,1,distance_from_energy,8.577317," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +19,1,distance_from_targets,7.186394," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +19,1,redundancy,0.0," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +19,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +20,1,distance,Float32[0.59999955]," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +20,1,distance_from_energy,10.01089," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +20,1,distance_from_targets,5.405971," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +20,1,redundancy,0.375," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 +20,1,validity,true," +Convergence: ✅",Californian Housing,wachter,JEM,1,0 diff --git a/artifacts/results/images/gmsc_benchmark.png b/artifacts/results/images/gmsc_benchmark.png new file mode 100644 index 0000000000000000000000000000000000000000..489749f04574032a8170c565edf000cfca2775a3 Binary files /dev/null and b/artifacts/results/images/gmsc_benchmark.png differ diff --git a/notebooks/gmsc.qmd b/notebooks/gmsc.qmd index fa63937a6b0051f1734b6ab2c2cfab5e78099993..75bb6b3f0a6bb9b42c7c9b767670e7651b4637f7 100644 --- a/notebooks/gmsc.qmd +++ b/notebooks/gmsc.qmd @@ -167,12 +167,12 @@ bmk = benchmark( factual=0, target=1, initialization=:identity, ) -CSV.write(joinpath(output_path, "gmsc_benchmark.csv"), bmk()) +CSV.write(joinpath(output_path, "gmsc_benchmark.csv"), bmk.evaluation) ``` ```{julia} -@chain bmk() begin +@chain bmk.evaluation begin @group_by(dataname, generator, model, variable) @summarize(mean=mean(value),sd=std(value)) @ungroup @@ -182,14 +182,15 @@ end ```{julia} -df = @chain bmk() begin +df = @chain bmk.evaluation begin @filter(variable in [ "distance_from_energy", "distance_from_targets", - "distance",]) + # "distance", + ]) @mutate(variable = ifelse.(variable .== "distance_from_energy", "Non-Conformity", variable)) @mutate(variable = ifelse.(variable .== "distance_from_targets", "Implausibility", variable)) - @mutate(variable = ifelse.(variable .== "distance", "Cost", variable)) + # @mutate(variable = ifelse.(variable .== "distance", "Cost", variable)) end plt = AlgebraOfGraphics.data(df) * visual(BoxPlot) * mapping(:generator, :value, row=:variable, col=:model, color=:generator)