,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
254,0,Finetune Model,Llama-2-7B,0.0,0,-1,1e-05,0.0,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9844665453461495,0.0051727714413351816,0.5021345577886491,0.013075097432894608,0.9905248646281772,0.0007461848129786882,0.9876392022245997,0.004309388867036841,0.9158566386007846,0.0035939738499808516,0.38,4.709184258054772e-44,0.036345514950166115,,,,,,,,,,1.096624314778916e-19,0.6285644571327744,1.096624314778916e-19,Baseline
153,0,Grad. Ascent,Llama-2-7B,0.0,1,0,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
154,0,Grad. Ascent,Llama-2-7B,0.0,1,1,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5900429345628848,0.033905759300725195,0.4547378833757618,0.022335305669671995,0.1755233138034995,0.007054415039534881,0.9705581657460249,0.005055939888176935,0.8874643874643875,0.02809493708284436,0.5601436387215902,0.03594055775086999,0.4311578250642015,0.026881223608336224,0.101544103709312,0.006030005405901275,0.9125813125165163,0.0071185045685729365,0.9889397136536354,0.0037059751472614064,0.48344775524839767,0.01405782374996479,0.9872371886220882,0.0014907038206082325,0.987697022848052,0.004142802637345347,0.9232821917154683,0.0034525307216189344,,,,0.7761951905387826,0.03772874843774116,0.5348676866651111,0.034412833437561345,0.8994496957903285,0.01639860932264404,0.7697821692084698,0.03782303934764471,0.9036587801876312,0.009903910085434673,0.425,3.489894879749045e-07,0.08048780487804878,,,,,,,,,,0.0012708143485281624,0.6298261119892148,0.0012708143485281624,Baseline
155,0,Grad. Ascent,Llama-2-7B,0.0,1,2,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5873988265455783,0.03397508491678948,0.4508092646908872,0.022551229274736694,0.1739081567383425,0.007062546425329464,0.9665255339610767,0.005908090892117214,0.8760683760683761,0.029119795619916317,0.5583610983349427,0.03599265069970819,0.4281143594726287,0.026936659748423613,0.09949787624348999,0.005927416138043877,0.9107472606370924,0.0070831369098684925,0.9829393276544355,0.004747087133082975,0.4874752426550318,0.014033035101973904,0.9855465733622553,0.0018392288196213203,0.9823404390280708,0.0048268090647223105,0.9234222857023673,0.00337002024051439,,,,0.6634498511248423,0.03979501310983266,0.5330159294477221,0.03463020113629034,0.7603332595788647,0.033966656011972296,0.6583112962777088,0.039468344041597575,0.9078713971845589,0.009329761694848934,0.425,2.0396328181959689e-07,0.07439024390243902,,,,,,,,,,0.0012708143485281624,0.6274349162486959,0.0012708143485281624,Baseline
156,0,Grad. Ascent,Llama-2-7B,0.0,1,3,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5767494734917964,0.034372249724000214,0.4408938678030151,0.02287028723794112,0.1750376753678611,0.007158422081483091,0.9663828541959902,0.00575613164904337,0.8917378917378919,0.02791127949453827,0.5513137330681693,0.03617204964693906,0.42021024790475425,0.027096090976780653,0.09769306113650794,0.0059427576527659514,0.9055722933002827,0.007772959154843439,0.9616573619305047,0.006924760552611335,0.4892999713770692,0.014168689417265349,0.9774506636931869,0.0028459379602567628,0.9583282920447302,0.007198221797581762,0.9220386919418376,0.003423647927837329,,,,0.5121226937822042,0.032962969948126364,0.54345131884367,0.034968297994459455,0.43023176455719464,0.04345304441763552,0.470313608214277,0.03389656752927492,0.8711500171294493,0.010297842300251473,0.375,5.841629899805412e-07,0.08658536585365853,,,,,,,,,,0.006760732303569208,0.6208832381499613,0.006760732303569208,Baseline
157,0,Grad. Ascent,Llama-2-7B,0.0,1,4,1e-05,0.98989898989899,0.943,0.02184680361707013,0.569912690980619,0.03466840517291862,0.43204434512382306,0.023307965482841985,0.17792390999093788,0.007721143710477113,0.9666332582272671,0.0057618895495265676,0.8831908831908832,0.028916035930034605,0.5473668144428057,0.03622314683593883,0.414897165484955,0.027344081869356943,0.09466068410867952,0.006050371078030876,0.8990135186717393,0.007874996621043741,0.9360858953948274,0.009325230518351905,0.49302946184701624,0.014332776372096316,0.9674047600471735,0.003620235746904036,0.9232295288474817,0.01000286376090822,0.9195202169991762,0.0035090468594729846,,,,0.4726138841044719,0.03209104244253,0.5596537628394563,0.038582904584907325,0.273127242585076,0.040910247561870554,0.4193707732983415,0.03496432463333179,0.8648833328747095,0.010606785895688082,0.35,2.8431650207494386e-05,0.14146341463414633,,,,,,,,,,0.01430154804770646,0.6152017367360003,0.01430154804770646,Baseline
133,0,Grad. Diff.,Llama-2-7B,0.0,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
134,0,Grad. Diff.,Llama-2-7B,0.0,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5831342368229205,0.034195526749926465,0.4463399542137994,0.022676754446149583,0.1741332194637148,0.007005056236897536,0.9711427967298144,0.004892496292632562,0.8746438746438747,0.02986595450792445,0.5553851084435623,0.03611651818945058,0.42606843965321073,0.02705610226448954,0.09996325669567287,0.006159195821413578,0.9108871819027446,0.007062997168334457,0.9881749522878556,0.00403912406410768,0.48451691239938044,0.014172133539883811,0.9886402516003234,0.0015158301664681946,0.984391926889841,0.00497080235933812,0.9225775765926555,0.0033693716971956124,,,,0.8470500248532187,0.03425612789078735,0.5321108595821069,0.0345352995124133,0.9437253641548825,0.010302771440246095,0.8551610918357241,0.0338847158578802,0.9129679236930579,0.008755089796400987,0.425,3.489894879749045e-07,0.08048780487804878,,,,,,,,,,0.0012708143485281624,0.6247652337390697,0.0012708143485281624,Baseline
135,0,Grad. Diff.,Llama-2-7B,0.0,1,2,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5743689863204028,0.03459778800824567,0.4395178398225532,0.02291569750507518,0.17573586844871675,0.0071367315805213145,0.971376699089106,0.004833659056828999,0.8846153846153846,0.02814097500969468,0.5500806928206896,0.03629220886455765,0.4206527134660717,0.027177369607437277,0.10089550395690719,0.005891426629145233,0.9103603531622805,0.007225732932586464,0.9876166503747148,0.003958159615863906,0.48619738136346835,0.014218352461945517,0.9870674423893376,0.0017276553352510255,0.9834584001935912,0.005012665963437899,0.9230005342696567,0.003320496577337283,,,,0.693891532134268,0.040761864873975456,0.5313708135093245,0.03468764401794746,0.7907704753851272,0.03193867027599963,0.697210810027939,0.0408282479347697,0.9117740693087659,0.008923458576449092,0.425,3.1401395972352475e-07,0.07926829268292683,,,,,,,,,,0.0012708143485281624,0.620856677526391,0.0012708143485281624,Baseline
136,0,Grad. Diff.,Llama-2-7B,0.0,1,3,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5588856115278307,0.03486192436995179,0.42454388004631033,0.023382520023456987,0.17113257408226445,0.006696328375983813,0.971384233041181,0.004872506472042895,0.8660968660968662,0.030161453599377153,0.5440843064984168,0.03625778397852338,0.4103980888734902,0.02752113340668667,0.09743177268489672,0.006137860799523201,0.9075885179544705,0.007384150110357787,0.9567632211441702,0.008380284589874188,0.4919675035655331,0.014354171258223164,0.9788901898126867,0.002587745577928834,0.9498857465693834,0.009150898069415532,0.9209175410960485,0.0035039275462926496,,,,0.5611743395394859,0.0361031249078284,0.5357745264531568,0.035138959954993894,0.48186478504437824,0.04263428759390712,0.5391800516220583,0.0371989002115956,0.8954952252076869,0.010479046564108179,0.4,1.055741449818015e-06,0.09390243902439024,,,,,,,,,,0.003018184077228396,0.6098886454382855,0.003018184077228396,Baseline
137,0,Grad. Diff.,Llama-2-7B,0.0,1,4,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5495203912493695,0.03510662588135505,0.41377046154740876,0.023744863866831466,0.16752673547642588,0.006608755494593228,0.9705274573313899,0.005449214872377944,0.8703703703703705,0.03001669034410197,0.5372712890340352,0.036215753077327884,0.40173797204393547,0.02764724919841151,0.09540026299698359,0.006152647895052731,0.9050756487748767,0.007502582140361348,0.916460247115961,0.010795592976781397,0.49330805534207384,0.014632422453902047,0.965061026568477,0.0034352591571064324,0.9011082675444243,0.011800933253074398,0.9188436974164093,0.0036434938209770657,,,,0.49253655875189073,0.03322129247825073,0.542568072727047,0.03635081642015713,0.2987510252217606,0.040704356395482096,0.442433429498021,0.03283629382771646,0.8643185765471209,0.012045353707243911,0.35,1.6715837773517706e-05,0.1329268292682927,,,,,,,,,,0.01430154804770646,0.601047475651797,0.01430154804770646,Baseline
163,0,KL Min.,Llama-2-7B,0.0,1,0,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
164,0,KL Min.,Llama-2-7B,0.0,1,1,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5926914370142016,0.03380708048334772,0.4552599427511621,0.022397353324240442,0.17548038451057024,0.007054889176896772,0.9701414990793583,0.005048595089360008,0.8917378917378919,0.02791127949453827,0.5606620010502634,0.03589288783261826,0.4313463368160679,0.026893905130676334,0.10109508486948855,0.006048669438358328,0.9115536136563003,0.007054328446766984,0.9866427948861284,0.004164280248771519,0.48380044848689596,0.014028587577548995,0.987079635387015,0.001524838575423045,0.9856550808912471,0.004541405936506324,0.9234270631972602,0.003406348491488958,,,,0.7786513675263843,0.038214845616106496,0.5342827939227608,0.03416607249348546,0.9022485278034719,0.01636718106068851,0.7863602232753086,0.03872457119116467,0.9052995628926588,0.009539580654677837,0.425,4.299272404750809e-07,0.08292682926829269,,,,,,,,,,0.0012708143485281624,0.6305829841958074,0.0012708143485281624,Baseline
165,0,KL Min.,Llama-2-7B,0.0,1,2,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5852513278783095,0.03415700434871471,0.4512026997232316,0.022526850296365288,0.1744369057670915,0.007011106460219607,0.9684490075591609,0.005697520028061312,0.878917378917379,0.02908252649809801,0.5583273505371179,0.03607477677978727,0.42904895977528,0.027004029214984314,0.10027173126835717,0.006095704998244655,0.9102069325105575,0.0071130182494677984,0.9837020298645062,0.004679527798699983,0.4863827807838256,0.014036072547985811,0.9851094086008342,0.0019048629397133007,0.97964844544246,0.005470548235405333,0.9232841112517135,0.003354321867024587,,,,0.6452102127494272,0.038795734192378806,0.5339878688212196,0.03469282005007738,0.7561577607136813,0.033841205404102265,0.640837113817565,0.039597880960097746,0.9005021575058872,0.009514024034688795,0.4,1.634398358874023e-07,0.07195121951219512,,,,,,,,,,0.003018184077228396,0.6274392149443913,0.003018184077228396,Baseline
166,0,KL Min.,Llama-2-7B,0.0,1,3,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5771976816818047,0.034452997218025624,0.4406256089564199,0.022935863913454847,0.17579360511063297,0.007156745309643807,0.9661798328164983,0.005836219904184736,0.878917378917379,0.02908252649809801,0.5505496096178601,0.036126357568608575,0.42045603527182135,0.02714191375993806,0.09428626024741103,0.006039913345791347,0.9011634386488793,0.007877591930412776,0.9631817083850834,0.006755508048376161,0.4901608167131136,0.014162208578683038,0.9758064280747213,0.0031446263998563873,0.9574773675645152,0.007325667723048805,0.9220544162502203,0.0034188156565595406,,,,0.5208700880045747,0.033876535953684686,0.5440980827111798,0.035182167822362186,0.42186229134424025,0.04294040838303995,0.4713160054717275,0.03472033829656913,0.8696356371868526,0.011975877633282655,0.4,1.6943467926466838e-06,0.1,,,,,,,,,,0.003018184077228396,0.6202838445189751,0.003018184077228396,Baseline
167,0,KL Min.,Llama-2-7B,0.0,1,4,1e-05,0.9746444032158318,0.943,0.02184680361707013,0.5692502257120079,0.034771819647641575,0.43256772458393544,0.0233634075991481,0.17679272538606905,0.007668194919986859,0.9662452463366895,0.006148103128195363,0.878917378917379,0.02908252649809801,0.5470188356526116,0.0362099382523936,0.4138519074688882,0.027345791961117884,0.09082474692862029,0.005808831078849859,0.8958202632877043,0.008035621885511554,0.9400564537393836,0.009134905713481647,0.49234372072457055,0.014370784864646631,0.9647276478070979,0.0039017286602883216,0.9235177368548603,0.01012576780838096,0.9189732932079522,0.003530267429751703,,,,0.47597794758075834,0.03239373479241791,0.558581347045137,0.038240055904861195,0.2621098289071951,0.03934582654858117,0.40951543215189046,0.030112049427227196,0.8638400038292596,0.010322399053324752,0.375,2.8431650207494386e-05,0.14146341463414633,,,,,,,,,,0.006760732303569208,0.6146480741594899,0.006760732303569208,Baseline
143,0,Pref. Opt.,Llama-2-7B,0.0,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
144,0,Pref. Opt.,Llama-2-7B,0.0,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5924105428404896,0.033263694434418234,0.4565522451578235,0.021610794050383905,0.1766362229531734,0.007145618092346133,0.9709098974802777,0.0050599919253484765,0.8874643874643875,0.02809493708284436,0.5594055257705869,0.03567260486346218,0.43350943486503823,0.02661437028896882,0.1056862156657845,0.006183210785840322,0.9198611150631801,0.006919614182981682,0.9810836592209436,0.005358997022471166,0.47823084373482894,0.013991136456899415,0.987682564962214,0.0014294465117274482,0.9837301765046863,0.005270907546412638,0.9240710251171401,0.003335151516141506,,,,0.9685243855606759,0.017780552723972488,0.5416122420147298,0.0335048502246019,0.9917345789108578,0.0011589828601444156,0.9660224780701754,0.018520210577111264,0.9166642498201292,0.008456217893549868,0.425,1.6943467926466838e-06,0.1,,,,,,,,,,0.0012708143485281624,0.6296293128306515,0.0012708143485281624,Baseline
145,0,Pref. Opt.,Llama-2-7B,0.0,1,2,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.606546647433683,0.030961077053547004,0.46835196679220437,0.019662428798979843,0.18027710108089362,0.007120540722361764,0.9711053228188717,0.005193141445257983,0.8717948717948718,0.029276166353085355,0.5598299511583128,0.03513500053040345,0.4438246441699819,0.025308810405222027,0.10646149121607264,0.006189134075678301,0.9216962290636657,0.006709124114274743,0.9719699980936025,0.0063748268799224865,0.468221641983294,0.013605114756446606,0.9822887429450634,0.0014869938769852837,0.9847992127106778,0.0049558019543791566,0.9257162977066123,0.003308987703675799,,,,0.8186714818241565,0.043115858610446985,0.566483639868165,0.03249111598229366,0.9565788230001161,0.005529587923744388,0.8353519284034615,0.042068217012389,0.9176329318591485,0.008773384722766613,0.425,4.5276301534613594e-06,0.11341463414634147,,,,,,,,,,0.0012708143485281624,0.632871908926051,0.0012708143485281624,Baseline
146,0,Pref. Opt.,Llama-2-7B,0.0,1,3,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.6209160392815353,0.02921498876116747,0.47751150900909095,0.018195830443283427,0.18402368903026795,0.007452606075757043,0.971230589018051,0.005180439432376149,0.8660968660968662,0.03016145359937715,0.5636513328534187,0.03397030619562475,0.4500346651850733,0.024116036374058982,0.10894654161447405,0.006392292787809994,0.9267264216214894,0.006596954642520662,0.9394917009335717,0.010209371403798576,0.460885816164811,0.013399740935308158,0.9751565902878809,0.0017260002980401803,0.9733056382182119,0.006991045991164258,0.9287424401721425,0.0033082966207093195,,,,0.5236425314354136,0.06006007523324811,0.5820412789879955,0.03188653465299932,0.9042981066202153,0.008310046664909112,0.5434620650095046,0.061323014422537105,0.9316512654147395,0.010667582742999213,0.375,5.845156920258887e-06,0.11707317073170732,,,,,,,,,,0.006760732303569208,0.6345567312241833,0.006760732303569208,Baseline
147,0,Pref. Opt.,Llama-2-7B,0.0,1,4,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.6226602095796977,0.028556734993852778,0.479269047982605,0.017397555112613837,0.19010803523961417,0.007968879579021804,0.971887674038294,0.005207517121420166,0.8917378917378919,0.02724332888874447,0.5622231069115052,0.0336037536961641,0.45085733177400755,0.02340373364616858,0.11387149699825733,0.0061898143954831585,0.9334525928992325,0.006707866917734486,0.905155660984239,0.012720361050411409,0.4567800978942038,0.013256573510614282,0.9686896057092785,0.0019275416460468816,0.961450621340768,0.008536871990541558,0.9323639006302699,0.0032033723924312922,,,,0.15229945036720657,0.049092941234932366,0.5901997851872188,0.031438784846847985,0.8591139219396796,0.00928767343239399,0.18063505337112148,0.0493767152706705,0.9691722958986757,0.010434715487281379,0.375,1.6715837773517706e-05,0.1329268292682927,,,,,,,,,,0.006760732303569208,0.6335834527767225,0.006760732303569208,Baseline
169,0,Retain Model,Llama-2-7B,0.0,1,-1,1e-05,0.0,0.945,0.02105188395801711,0.5926241084713598,0.03345059593050104,0.4544514898987734,0.021893655065203167,0.17353989749500584,0.006538369470852875,0.967232018904305,0.005690027028802709,0.878917378917379,0.02908252649809801,0.5442517457082237,0.03644994289779965,0.4220565650208274,0.027247431184912106,0.10057341165244862,0.006012069013665042,0.9135588025349304,0.006767434505349635,0.9809455309460164,0.0058307259379082295,0.47882616449120396,0.01410518223667099,0.990023753985948,0.0007292641153727538,0.9846272824265296,0.005002917384509446,0.9241821753906638,0.0033307787579352774,,,,0.4057248745903669,0.029331873414771226,0.6701244699861956,0.02808055457287809,0.1736882915925146,0.025915814128189336,0.4039820476083342,0.03168689004734824,0.8936834437422675,0.012238378133721525,0.0,1.0,0.0,,,,,,,,,,1.0,0.6246309817225171,1.0,Baseline
148,0,Grad. Ascent,Llama-2-7B,0.01,1,0,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
149,0,Grad. Ascent,Llama-2-7B,0.01,1,1,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5800258532067911,0.034048775490725,0.4479316827635683,0.02371490849542139,0.17520921205557424,0.007085165889905948,0.9712346794000724,0.004948616955739485,0.8831908831908832,0.02891603593003461,0.5605509089165185,0.03578007402333147,0.4380829002046114,0.0319031334543632,0.09930950443878336,0.006054405871120595,0.9114705966643226,0.00693546302693967,0.9812901725231946,0.005294521902057731,0.48294462031737356,0.01404737202686901,0.9880106777149297,0.0009239754509367609,0.9815029129733047,0.005299277163453248,0.9239376796665825,0.003345169932380108,,,,0.7977512522720149,0.038099059434458245,0.5338572177221842,0.032619997082725165,0.9015410916473158,0.017060016135887153,0.7871158272978711,0.03931892842575011,0.9099585668064352,0.010321529594744945,0.45,0.0011065220933232922,0.2121951219512195,,,,,,,,,,0.0005039436209702519,0.6280934372877442,0.0005039436209702519,Baseline
150,0,Grad. Ascent,Llama-2-7B,0.01,1,2,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5743467636338636,0.03437254720889397,0.44299784489058774,0.023919808917236895,0.17549321155280903,0.007031472326137827,0.9716746149878339,0.004948396958023329,0.8874643874643875,0.0287431042820868,0.5528340897493834,0.03593856509215231,0.4337222544394661,0.03205121933349278,0.09969244858530825,0.006010696651478088,0.913665955907049,0.006868370009154204,0.9769308651638873,0.005956200936345651,0.4843319150944858,0.014081501536447579,0.9873622386393629,0.001058850510217853,0.9770827849479735,0.005975149733698831,0.9237527444463929,0.0033456930924993327,,,,0.6770552495067294,0.04141887944577475,0.5313505413009971,0.032377653394326515,0.7863983416805109,0.03208577061571647,0.6732327143425557,0.04134044820373354,0.9030402645531372,0.01113914903642642,0.45,0.0007984068197401939,0.2048780487804878,,,,,,,,,,0.0005039436209702519,0.6244505684885571,0.0005039436209702519,Baseline
151,0,Grad. Ascent,Llama-2-7B,0.01,1,3,1e-05,0.98989898989899,0.9229999999999999,0.02553883950441608,0.5605770276733398,0.034814648533758916,0.4302945032709254,0.024239223157741678,0.17157705989401034,0.006932316728979308,0.9699656201319954,0.005618235030283827,0.8660968660968662,0.030766115083144873,0.5465426863610293,0.03605363485526744,0.4271626730893178,0.032234933455384716,0.09585526374650297,0.006212504348587216,0.9066163830397892,0.007232014686092795,0.952820588478855,0.008377862576579216,0.48822552142839126,0.01408030508332057,0.9795794731504118,0.0020893988669301346,0.9511125315484827,0.008266508391635445,0.9215150317951316,0.0034707050691802194,,,,0.5415403816676896,0.03633105352195194,0.536498562129631,0.034032965640264846,0.45397324031395614,0.041194928318438455,0.5190002196868267,0.036133109171101345,0.8924787780443587,0.013457525433243911,0.425,0.0009936344267771346,0.2097560975609756,,,,,,,,,,0.0012708143485281624,0.6149556371319791,0.0012708143485281624,Baseline
152,0,Grad. Ascent,Llama-2-7B,0.01,1,4,1e-05,0.98989898989899,0.9229999999999999,0.02553883950441608,0.557230500186014,0.034865896707917045,0.4258273633007778,0.024371014502785608,0.1694671556041842,0.006915481170274377,0.9687123356509246,0.006122574201166815,0.8618233618233619,0.030902274103896878,0.5440057886627606,0.03608321893084478,0.4248389901132918,0.03233250572807769,0.09186920722354087,0.006054334602349437,0.9016770263139969,0.007509414988868026,0.9414179134529468,0.009380750631925328,0.4891483824586931,0.014149859802891799,0.9757943241994819,0.0024221258047436266,0.9382342689565394,0.009331698800946149,0.9206883117273208,0.0035053740277021716,,,,0.515438355924935,0.03450413884417871,0.5349858627805251,0.03441539018972547,0.3484092705229157,0.03912387599627555,0.4880769534734265,0.03613318624504579,0.8743174909059739,0.012123663709260601,0.4,0.0013674832061951747,0.21707317073170732,,,,,,,,,,0.003018184077228396,0.611821828661174,0.003018184077228396,Baseline
128,0,Grad. Diff.,Llama-2-7B,0.01,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
129,0,Grad. Diff.,Llama-2-7B,0.01,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5773765341443244,0.03417794235272885,0.4441402807130932,0.02402033482691025,0.17514580487452006,0.007137633259603525,0.9715280541412732,0.004854702506416361,0.8746438746438747,0.02986595450792445,0.5548336070164175,0.03585489922508421,0.4340930797609021,0.032090674904577,0.10036751772585996,0.0062275981311647145,0.9128550837341167,0.006846382858297305,0.9812212692042912,0.005429907335500562,0.4828983996735526,0.014174579326499147,0.9889669057437027,0.0008745702996192081,0.9769337794228795,0.006224395582782556,0.9225404481429507,0.003328455303415582,,,,0.8390636054175344,0.03505917880239604,0.5313141361531593,0.03296334226008337,0.9427974085006843,0.011427898495431734,0.8358652145198164,0.03613972046757268,0.9157937185986256,0.010161506733134736,0.425,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.0012708143485281624,0.6247498581220056,0.0012708143485281624,Baseline
130,0,Grad. Diff.,Llama-2-7B,0.01,1,2,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5692104903614713,0.03439893577976867,0.4361785377749471,0.02410946926548582,0.17353800989172505,0.007043791207283828,0.9731467522708408,0.004779718421559819,0.878917378917379,0.029709155181433657,0.5485601874923083,0.036093590198850146,0.42982827628427805,0.032163414601145704,0.09954316594928822,0.006028171804746514,0.9099425999261167,0.007378812492078975,0.9822425534755755,0.005285317705376488,0.48467084926549414,0.01420301034167578,0.9879979225459306,0.0010270337380533711,0.9772090305588494,0.006242996332846527,0.9224518287464465,0.0033328897682729734,,,,0.7215835454295053,0.043592000438350396,0.5276723573440643,0.03284856684429458,0.8300386090545793,0.028438250445988612,0.7318984963117551,0.043410059838410925,0.9098268541289307,0.010555482868827449,0.45,0.0007553515715699177,0.20365853658536584,,,,,,,,,,0.0005039436209702519,0.6206049086240365,0.0005039436209702519,Baseline
131,0,Grad. Diff.,Llama-2-7B,0.01,1,3,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5574112343706722,0.03481966255137871,0.4228619983745002,0.024639522296912447,0.16625247534991444,0.006622426447691064,0.9726272150051648,0.005160253129210175,0.8660968660968662,0.030766115083144873,0.5414314892361967,0.0361188042720218,0.42204842880404664,0.0325687219555866,0.09611545317402165,0.00602262255391896,0.9086252009559641,0.007404644959587136,0.9550050353791786,0.008610146710774336,0.4872969147494578,0.014309154781936713,0.9810517654599463,0.0018522824870059965,0.9479980529273969,0.00914181826143523,0.9208812427369228,0.0033613659941016034,,,,0.5765635074871464,0.03586214752395286,0.5253032114387033,0.03385578852277948,0.5372222789151637,0.04213828810721073,0.5501486653771084,0.035514857396460875,0.8946132097747274,0.011117196442996429,0.425,0.0005381573264457984,0.19634146341463415,,,,,,,,,,0.0012708143485281624,0.6109236267561884,0.0012708143485281624,Baseline
132,0,Grad. Diff.,Llama-2-7B,0.01,1,4,1e-05,0.9837146980004123,0.9179999999999999,0.02683206283459391,0.5520435933650645,0.03481328532455935,0.415991757729618,0.024877486524841206,0.1653701412476612,0.006760408232245415,0.9722346052249344,0.005086449152721892,0.8660968660968662,0.030766115083144873,0.5371617229622194,0.03607485029885521,0.4187476835115922,0.032689354178344066,0.09600424215333588,0.00599081988604742,0.9103354497720064,0.00732678373911513,0.9400970491453476,0.0097824948523407,0.48856732664519475,0.014411133751347567,0.9759288396427315,0.0021738910844479944,0.9302713108973918,0.01033028178851249,0.9200524749123263,0.0033835639324742406,,,,0.5084746164474694,0.03774565816012248,0.5240481718915301,0.035010545007557396,0.40058082944838047,0.04163737641558911,0.5073350257593472,0.03910212571406874,0.8901339308097882,0.011020560726255227,0.425,0.0008912169196264585,0.2073170731707317,,,,,,,,,,0.0012708143485281624,0.6062986298187203,0.0012708143485281624,Baseline
158,0,KL Min.,Llama-2-7B,0.01,1,0,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
159,0,KL Min.,Llama-2-7B,0.01,1,1,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.580110523651442,0.0342139237242958,0.44809062095334956,0.023725648301852434,0.1755352610066232,0.007067444719009651,0.9712346794000724,0.004948616955739485,0.8874643874643875,0.0287431042820868,0.5598849904079047,0.03575319998562076,0.4386095190114807,0.03188548587354564,0.0995082966488098,0.006022209617325384,0.9111060994449297,0.006978441012047056,0.9794719907050128,0.005675897883214464,0.4827531596223028,0.014049862186615527,0.9880365938733333,0.0009178672836727823,0.9804534198307884,0.005435411049767168,0.9242813685944077,0.0033437399379373286,,,,0.7950726808434434,0.03891439833031167,0.5335169256291284,0.03239227876815981,0.9014313140700427,0.017028902783455285,0.7837473656468008,0.04010397898938578,0.9104373329081868,0.010183776798575905,0.425,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.0012708143485281624,0.6282876063598299,0.0012708143485281624,Baseline
160,0,KL Min.,Llama-2-7B,0.01,1,2,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5754402690990706,0.03430368893520067,0.44406569523416695,0.023922200543703864,0.17622850567045614,0.007070840040822426,0.9734853810333826,0.0048032052840292685,0.8831908831908832,0.02891603593003461,0.5539892135948579,0.036011802420717914,0.43446712016321604,0.0320295895982226,0.09881985331421088,0.006023363827002265,0.9090761817520574,0.0072184737616799765,0.9773090164243915,0.005847060879197442,0.484487386660019,0.014074711958540534,0.98738023909771,0.001054780870647492,0.9764651020476818,0.006139645378646104,0.9233519772587567,0.00336247411780429,,,,0.6796703091782407,0.04243778859675377,0.5311777081329236,0.03249675958188966,0.786087349932698,0.03206775939949202,0.6741205310232469,0.04218301837400526,0.8993946675439609,0.011606517481817501,0.45,0.0009411722930963151,0.20853658536585365,,,,,,,,,,0.0005039436209702519,0.6249746605306692,0.0005039436209702519,Baseline
161,0,KL Min.,Llama-2-7B,0.01,1,3,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5613741848967867,0.03464862221361874,0.43026663892093714,0.02419244482979826,0.17626374559393135,0.007117271946580789,0.9703688320138824,0.005590378744295158,0.8660968660968662,0.030766115083144873,0.5470421743758249,0.035915415061641885,0.426414986253531,0.032244399064120764,0.09518918609090538,0.006247104685379855,0.9064787748921043,0.007343742226863473,0.9530836815948891,0.008434206671982544,0.48817648773815203,0.014081933722265658,0.9797252028178876,0.0020761009586852067,0.952067666485085,0.008289357264083934,0.9216562188281612,0.003495696036059303,,,,0.5417339802648964,0.03560176457222332,0.5359991224635998,0.033713459909138134,0.45529475231527805,0.0412123298354884,0.5222129245706866,0.035585566289791426,0.8883371883219329,0.013243955948083337,0.425,0.0012308068326092325,0.2146341463414634,,,,,,,,,,0.0012708143485281624,0.6154517140995398,0.0012708143485281624,Baseline
162,0,KL Min.,Llama-2-7B,0.01,1,4,1e-05,0.9746444032158318,0.9229999999999999,0.02553883950441608,0.5589617571264943,0.034722005379876165,0.42686555147068356,0.024509365403373253,0.16860481894266724,0.006869397681122072,0.9711684239097478,0.005048161498283998,0.8618233618233619,0.030902274103896878,0.544318030740605,0.03604717566868592,0.4243098312756541,0.03234995486337138,0.0916169728987259,0.006034686543439203,0.901223023759475,0.007686433410408507,0.9430402910753245,0.00937070133163249,0.48913036359507317,0.01417050329800232,0.9761569459467297,0.002351134271672225,0.9412342727443869,0.009122881887916547,0.9206378021156858,0.0035345532864452995,,,,0.5197038216420136,0.03425461287449973,0.534672398512898,0.034361945689757485,0.3493818427001489,0.03916070926834415,0.48833168881585126,0.036106213139953505,0.8753322814923875,0.011951847964291485,0.4,0.0013674832061951747,0.21707317073170732,,,,,,,,,,0.003018184077228396,0.6123014101651115,0.003018184077228396,Baseline
138,0,Pref. Opt.,Llama-2-7B,0.01,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
139,0,Pref. Opt.,Llama-2-7B,0.01,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5818305995434252,0.0336258806698115,0.4516876545117113,0.022786998297976563,0.17735101469737688,0.007285532670262744,0.9709609804468069,0.004962772317491594,0.8746438746438747,0.029865954507924442,0.559317079149142,0.03573241714959832,0.4403488213603541,0.031356894081744115,0.10132621009432234,0.006328051690957193,0.912779227727123,0.00708526437373145,0.9809739936408115,0.005541810173179431,0.4769229838409399,0.013995921593912948,0.9884367518383954,0.0008572436618283804,0.9868234582761289,0.0044571911882983406,0.9248143893891777,0.0033181315019821128,,,,0.9624071550335319,0.01902751362238947,0.5445679585904102,0.03260085787341239,0.9902187818386323,0.00128667594293562,0.9679016436903499,0.018337879111251044,0.9172507689793704,0.008676551693240605,0.425,0.001770566779669025,0.22317073170731708,,,,,,,,,,0.0012708143485281624,0.6278577940849149,0.0012708143485281624,Baseline
140,0,Pref. Opt.,Llama-2-7B,0.01,1,2,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.5868525414917711,0.03227039377073378,0.4549850082151925,0.021227661982427308,0.17970599305235527,0.007227741865684971,0.9730753101674295,0.004857974780534291,0.8703703703703705,0.03001669034410197,0.554188936629353,0.03570215328899892,0.4434825705074438,0.030519104128676356,0.1040392353475466,0.006307929113897188,0.9189298043221829,0.007206825363839335,0.9703738069804334,0.00655653596424914,0.4711610329750893,0.013811135050113365,0.9861218016375611,0.0009899238404132483,0.9878782368309074,0.004284678608913816,0.9267550927503032,0.003260527718965425,,,,0.8363801365883157,0.04517391821008251,0.5571474547164842,0.03192519282255046,0.972944523546138,0.0039513920926697995,0.8405939756050049,0.043694964587067335,0.9191168798699401,0.009173316675796253,0.4,0.002638970910993521,0.2329268292682927,,,,,,,,,,0.003018184077228396,0.6273896698700678,0.003018184077228396,Baseline
141,0,Pref. Opt.,Llama-2-7B,0.01,1,3,1e-05,0.9837146980004123,0.9263333333333332,0.024519146644786646,0.6120984951453181,0.029602092247205808,0.4701069849132854,0.018763352188774993,0.179398180165208,0.007462407405966698,0.9719109744898553,0.005046110224429372,0.8746438746438747,0.02924268600158814,0.558692577179061,0.03470032908926651,0.45270895342466627,0.028971933358048183,0.10814599375745441,0.006333302982646953,0.9223662570782941,0.007279004718073149,0.9488138305990799,0.009263087395909031,0.46129806944852114,0.013487744052917184,0.977738637105947,0.0015008608589127987,0.980706964350727,0.006084155109445612,0.9294579680297151,0.0032532959867766285,,,,0.6071157085321186,0.055098423683929434,0.5789698666797543,0.031504356391908586,0.9111425180364293,0.008094541326696002,0.6247685301405814,0.05693647789581892,0.9083199808994191,0.011159154252143345,0.4,0.006955192886380246,0.25853658536585367,,,,,,,,,,0.003018184077228396,0.6325965973075246,0.003018184077228396,Baseline
142,0,Pref. Opt.,Llama-2-7B,0.01,1,4,1e-05,0.9837146980004123,0.9263333333333332,0.024519146644786646,0.612985074642076,0.029414999552961535,0.47193884066480796,0.018208375192243286,0.17961196637899424,0.007417501989608266,0.972482403061284,0.0050452315349812996,0.8660968660968662,0.030161453599377153,0.5611374865952636,0.03433914339069163,0.4553333687005546,0.02848497315421883,0.10880235579883603,0.006353917645501141,0.9243275141232049,0.007109171912797723,0.926926400731736,0.011283896071661435,0.45949966474157417,0.013422309069073818,0.9747282014283064,0.0016379979235600203,0.9758053680543032,0.006949235339699891,0.9309483108631779,0.0032548366189807136,,,,0.3999643294452556,0.06221025434190201,0.5839567149240932,0.03139728091889601,0.8851536619883351,0.008884138981633464,0.4390431857167497,0.06400171941745215,0.9295107228173798,0.01018216034602813,0.4,0.007917965558590367,0.2621951219512195,,,,,,,,,,0.003018184077228396,0.63185656470058,0.003018184077228396,Baseline
168,0,Retain Model,Llama-2-7B,0.01,1,-1,1e-05,0.0,0.925,0.02487849259781215,0.5862944163094279,0.03374892372874021,0.4505683636462643,0.022065610809776242,0.16783975493031222,0.006529491837121361,0.9700812534017699,0.005269946117413013,0.8703703703703705,0.03001669034410197,0.5471365216754651,0.03632183056706994,0.42191312801444286,0.027224296551869066,0.09780821686211255,0.006152149197443829,0.9107498232250473,0.007020623478948183,0.9852486009202834,0.004927942762345694,0.4782308271896145,0.014196414741213824,0.989928497616268,0.0007514551842063065,0.9870143086012513,0.0044681490010742096,0.9240008227178458,0.0033281752786762583,,,,0.40618862927483795,0.030258984159567487,0.6701777021800741,0.028498124312231716,0.17508310272500335,0.02529203048185361,0.3914341294336592,0.028405449629408948,0.8923673878445355,0.012032068634356475,0.0,1.0,0.0,,,,,,,,,,1.0,0.6220150094809674,1.0,Baseline