,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
254,0,Finetune Model,Llama-2-7B,0.0,0,-1,1e-05,0.0,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9844665453461495,0.0051727714413351816,0.5021345577886491,0.013075097432894608,0.9905248646281772,0.0007461848129786882,0.9876392022245997,0.004309388867036841,0.9158566386007846,0.0035939738499808516,0.38,4.709184258054772e-44,0.036345514950166115,,,,,,,,,,1.096624314778916e-19,0.6285644571327744,1.096624314778916e-19,Baseline
237,0,Grad. Ascent,Llama-2-7B,0.0,10,0,1e-05,0.8888888888888888,0.935,0.023066657908563763,0.639321986950863,0.02916578579046248,0.49363023437238207,0.019479741549558876,0.18557388294539998,0.006901852533361839,0.9727154498284629,0.005044630866742326,0.8917378917378919,0.02724332888874447,0.6127095763193304,0.03322809941963627,0.4842731636355073,0.024276507101671337,0.11074532236787658,0.006167478421663473,0.9267770167488393,0.007431012060224246,0.7590342636690868,0.015104796862593178,0.45736276021211847,0.012550775126599904,0.8708933450519903,0.005706962631198245,0.7573635388671619,0.015787196143047584,0.9075492349157196,0.004721891263704779,,,,0.6457007817080724,0.01610762667676562,0.5545876954996629,0.012162840068068682,0.8035631962554466,0.008238906949909844,0.6655017234877775,0.016125155740135698,0.9064357777501398,0.004641677751510154,0.3566666666666667,2.782883698771258e-32,0.10604651162790697,,,,,,,,,,2.4311282147882553e-17,0.6369385144270675,2.4311282147882553e-17,Baseline
238,0,Grad. Ascent,Llama-2-7B,0.0,10,1,1e-05,0.8888888888888888,0.0,0.0,0.4077049237168381,0.03628165591507205,0.36069190982075744,0.024137650326413854,0.0,0.0,0.03532359456803016,0.0012068335732052346,0.02564102564102564,0.013361698726531753,0.5079778921478196,0.037288366757854026,0.4203077857295317,0.027928109246203765,0.0006606436165278721,0.00035763159644924673,0.04940850274454852,0.0020135295140645015,0.010597673370638182,0.0015173990038315442,0.05133855551734056,0.005506196270702445,8.090435293707815e-05,3.0864850938709007e-06,0.0301060101111982,0.004881273104830249,0.025775763354218658,0.0006209066743241699,,,,0.0029423097963217197,0.000867011640988565,0.8962493448512298,0.006774631425767037,6.718819323213358e-05,1.874896747714176e-06,0.003832574922713051,0.0016073028124386672,0.015372930339597227,0.0004222928428896629,0.44333333333333336,1.198135062817154e-16,0.22411960132890366,,,,,,,,,,6.734929816796876e-27,0.0,6.734929816796876e-27,Baseline
239,0,Grad. Ascent,Llama-2-7B,0.0,10,2,1e-05,0.888888888888889,0.0,0.0,0.2741836623331792,0.030059255074221698,0.27833923394822074,0.017495014720653318,0.0,0.0,0.006008998562960845,6.924146556274578e-05,0.0,0.0,0.25432863580131854,0.02818214287563306,0.2667157503096089,0.01591846126419448,0.0,0.0,0.00913689425626929,0.0004257643243124427,0.023493130818058382,0.001956076264004178,0.08615103543616762,0.007887081023340977,1.1936792308068104e-08,1.4659269730384158e-09,0.0035191713327813604,0.00029562484029665087,0.006044902484358766,2.9540634293564736e-05,,,,0.019354788481028052,0.001735326529721448,0.8390837151829551,0.007958179745762918,1.581361441567216e-08,1.8285284943827604e-09,0.0037698676169052145,0.0004201102720891905,0.007671668427520525,0.0002501573549960421,0.37,8.242744297834301e-16,0.23187153931339977,,,,,,,,,,1.1626372139489807e-18,0.0,1.1626372139489807e-18,Baseline
240,0,Grad. Ascent,Llama-2-7B,0.0,10,3,1e-05,0.8888888888888888,0.0,0.0,0.3451118896848008,0.035262023738328825,0.2654862331095165,0.02441789024486365,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2759147386483986,0.03055101090664871,0.2659098112539255,0.019474405708078305,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.023493130818058382,0.001956076264004178,0.09280725881600144,0.008398505797577343,3.0265274212817936e-43,4.0805330524246387e-44,0.0035191713327813604,0.00029562484029665087,0.00601575039373741,8.670964358060814e-06,,,,0.021959685681935064,0.0017945257959866888,0.8227908656605494,0.008456339085038579,2.099497388827357e-43,2.8094940671826073e-44,0.004257163220541716,0.0004004390678119029,0.006318252700127254,1.88451654637476e-05,0.37666666666666665,1.5059847871718583e-16,0.22502768549280178,,,,,,,,,,2.4278206754998463e-19,0.0,2.4278206754998463e-19,Baseline
241,0,Grad. Ascent,Llama-2-7B,0.0,10,4,1e-05,0.8888888888888888,0.0,0.0,0.3269224085874874,0.03553888589519956,0.26087809301223186,0.022997732080621072,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2639029216517767,0.02970707212999506,0.25038824962550127,0.01808083054988123,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.023493130818058382,0.001956076264004178,0.0806006722921498,0.007850324845036232,2.4870485224433162e-45,1.059171467762705e-45,0.0035191713327813604,0.00029562484029665087,0.00601575039373741,8.670964358060814e-06,,,,0.021959685681935064,0.0017945257959866888,0.8212739142873943,0.008604361826885261,2.8280620064861e-46,2.1673711747569428e-46,0.004257163220541716,0.0004004390678119029,0.006318252700127254,1.88451654637476e-05,0.42,1.633819135526479e-19,0.19906976744186047,,,,,,,,,,4.283655759915899e-24,0.0,4.283655759915899e-24,Baseline
217,0,Grad. Diff.,Llama-2-7B,0.0,10,0,1e-05,0.8208616780045351,0.9329999999999999,0.02378587136868601,0.6017050644931898,0.03335182531000466,0.46439537840119605,0.021773959515579,0.17765810737250365,0.007036279399314868,0.9732819448390596,0.004818666476013726,0.878917378917379,0.02908252649809801,0.5658409833832219,0.03580461862859633,0.44140225723298576,0.02653909287117947,0.1001569247841251,0.006167591187626715,0.9120509105831243,0.006961436588866337,0.9439372420217687,0.009093638289382974,0.47874545153703413,0.013931154994749385,0.9650354158088728,0.004936831484319112,0.9395914319237352,0.009723657483135304,0.9218523281478709,0.0033207278998120707,,,,0.8279958948752504,0.014456977235174808,0.5132490834142889,0.012756328558632379,0.8957065615508024,0.008701539298098826,0.8325447152474319,0.014559500057591658,0.9101082074744462,0.0038004244112380676,0.37666666666666665,8.637897501068093e-43,0.043322259136212626,,,,,,,,,,2.4278206754998463e-19,0.631931019172421,2.4278206754998463e-19,Baseline
218,0,Grad. Diff.,Llama-2-7B,0.0,10,1,1e-05,0.8208616780045351,0.2395,0.041105415248340396,0.6637071906736927,0.03499756322151707,0.5006520611720955,0.028797052962651275,0.028884233085759534,0.007224694400450535,0.22013790127503782,0.03375559017310177,0.6994301994301994,0.040476382929679665,0.6049133725321804,0.0374065106054476,0.44824144713636704,0.031482607765591857,0.0508650510555151,0.005992038643453871,0.47176974083875073,0.03825299719404257,0.1940621873835773,0.010875469848437699,0.5321471219659181,0.018113136607973725,0.08029853094020394,0.009080763060991275,0.04763800890846115,0.007280554952068511,0.10376637940494497,0.009709885629396944,,,,0.13215956367882262,0.007819770118392487,0.47054546446276957,0.01637289783927431,0.007166722051071145,0.00135968194654159,0.02468846229015037,0.0022677787929059404,0.0681268081601237,0.0050731077593151215,0.25,5.996384025391428e-10,0.2938205980066445,,,,,,,,,,1.2161555648200794e-08,0.2771025634217845,1.2161555648200794e-08,Baseline
219,0,Grad. Diff.,Llama-2-7B,0.0,10,2,1e-05,0.8208616780045351,0.8246666666666665,0.03639667971191366,0.7716129342926635,0.02711101025983347,0.6003164629985872,0.024309391793527348,0.17576922671855386,0.011730252843826802,0.8211617860758591,0.032913073112558346,0.8653846153846154,0.030040574840406443,0.6684446219297923,0.03355799759772953,0.5126992678745153,0.02852838667106847,0.09916848166746946,0.006858865309475956,0.8557972865618257,0.019558937240033814,0.47969544932695857,0.013842136672871895,0.48188703341694994,0.016110805147948697,0.5925631857781514,0.014027208377181112,0.4383910172494805,0.014673764149263236,0.8447563148688527,0.01178886731313009,,,,0.003446182787247659,0.0012221743506500406,0.8308275025136231,0.009132774696651125,1.316783048410202e-21,1.3167830424737205e-21,0.0005873863210476489,0.00029127351759039187,0.006964915785590125,0.0004884943714379773,0.39666666666666667,3.679605031161672e-18,0.2106312292358804,,,,,,,,,,1.834066410994743e-21,0.6154339494150426,1.834066410994743e-21,Baseline
220,0,Grad. Diff.,Llama-2-7B,0.0,10,3,1e-05,0.8208616780045351,0.8131666666666666,0.03494206765535528,0.7480069105187852,0.026136715089160878,0.5772025995225366,0.022650921742400333,0.11292089825188573,0.01261212419238183,0.5291833561481321,0.044102985179051056,0.8960113960113961,0.027048070647329614,0.6322477052581024,0.035020164693400045,0.5023836500824538,0.02799525050187554,0.037521662649680244,0.004496158696346748,0.3876574419202505,0.03467385292788596,0.4650240833941574,0.013101390811192933,0.45960024644518266,0.015069442811097483,0.6395384508088131,0.011492965208914876,0.41162107420930594,0.014184510313843275,0.8264969965235899,0.01307511197309943,,,,0.0031728249741101635,0.001235705769494023,0.8316092700399621,0.010887294360710006,6.0654680819660474e-21,6.065468081963931e-21,0.0010086210478379312,0.0007767165306611857,0.008764553745154415,0.0024465447707545957,0.44,7.614304680318855e-20,0.19630121816168328,,,,,,,,,,1.7358479825434806e-26,0.6049515529149103,1.7358479825434806e-26,Baseline
221,0,Grad. Diff.,Llama-2-7B,0.0,10,4,1e-05,0.8208616780045351,0.8283333333333333,0.034603331810543515,0.7410314862814202,0.027159926837193992,0.5684894504120689,0.022519322161015026,0.1481309832420959,0.014754149148002164,0.6897638799331602,0.04093380288556581,0.8846153846153846,0.02814097500969468,0.6217541529107024,0.03511705206157131,0.49794985516240536,0.027418583340325216,0.06700852726724865,0.006486128807716383,0.6054779949138777,0.0360060131709548,0.4842905671626088,0.013429933516760492,0.465500087751493,0.014744568528127373,0.6982811478097758,0.01057850117480759,0.4870599980430448,0.014620357971819296,0.868222858607831,0.011124107352862036,,,,0.0027645562402600346,0.0009677891504124913,0.8253288276203347,0.011211322478756376,2.569464885420393e-11,2.5694648854203926e-11,0.0009713940043384862,0.0007183837956173646,0.008989575964161063,0.0025442943262366513,0.44666666666666666,5.250991940257773e-22,0.17882613510520487,,,,,,,,,,2.5915513350347427e-27,0.6119129436260805,2.5915513350347427e-27,Baseline
247,0,KL Min.,Llama-2-7B,0.0,10,0,1e-05,0.7210884353741497,0.9329999999999999,0.02378587136868601,0.6306949821504628,0.02984444022555427,0.48812065139909044,0.019792072246190062,0.18448492615056084,0.007111793509795778,0.9717949429160205,0.005080787444463876,0.8917378917378919,0.02724332888874447,0.6089196326278249,0.03324161675158292,0.47949256076665053,0.024630685713867822,0.10960984316297058,0.0061094806603277365,0.9243883168985222,0.0075981688297105865,0.7818301121403088,0.014749426504914637,0.46137962375818975,0.012644891147924732,0.8834278628298303,0.005759935798228901,0.7810207291786861,0.015253341562826754,0.9075510769164923,0.004977618233046828,,,,0.6609828216295157,0.016228915751494665,0.5499371387027171,0.012163377475622435,0.8121718239175666,0.008389367755390367,0.679321206452044,0.016180928783190336,0.9079130568889912,0.004685132778857708,0.36666666666666664,7.235152077098904e-34,0.09596899224806202,,,,,,,,,,2.514967868068585e-18,0.6367781166414765,2.514967868068585e-18,Baseline
248,0,KL Min.,Llama-2-7B,0.0,10,1,1e-05,0.7210884353741497,0.0,0.0,0.5402461639245297,0.03600574225451481,0.4377429840336204,0.025980246926112623,0.0,0.0,0.023453994726261654,0.0013331858995269332,0.07977207977207976,0.02329757854837242,0.5158427438794723,0.035381660502371265,0.42315479058515443,0.026211373968099215,0.008279826397399883,0.0029537681708903156,0.049929865861088336,0.0020145590545926862,0.004545372564980409,0.0012725927467138094,0.06544738138766777,0.006211975083603987,8.76183301058252e-05,3.6862657482108066e-06,0.01267625945416885,0.005418806724771711,0.013704176635244118,0.00039420917812312814,,,,0.0019033486494012808,0.0007044513295972492,0.8873554552512541,0.006927552282255985,6.954328930705629e-05,1.9169546336514714e-06,0.003485845153332021,0.003333313810793645,0.012756444501676529,0.0001971283721271205,0.44333333333333336,1.7889808641066614e-17,0.21667774086378738,,,,,,,,,,6.734929816796876e-27,0.0,6.734929816796876e-27,Baseline
249,0,KL Min.,Llama-2-7B,0.0,10,2,1e-05,0.7210884353741497,0.0,0.0,0.2875923780255352,0.03864225077602472,0.22840392152875602,0.027473468846892658,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.4740902104755777,0.03836163076543421,0.3665297446688551,0.030570073966038922,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.002776942628903413,0.000977362983639031,0.1379782740604028,0.01266606859205146,4.625652861676623e-17,3.7230710863853035e-17,0.00018074200217366542,5.945931546389664e-05,0.006263598561709471,0.00011462215606194333,,,,0.0017985867446393758,0.0006771314754316643,0.6966231723053505,0.012175197411290807,2.376017642941821e-17,1.3289697435231252e-17,0.00017282752484634064,6.051558988995322e-05,0.012001646829836817,0.0003745878400720144,0.27,2.2855015436448923e-10,0.2888150609080842,,,,,,,,,,5.01791941596116e-10,0.0,5.01791941596116e-10,Baseline
250,0,KL Min.,Llama-2-7B,0.0,10,3,1e-05,0.7210884353741497,0.0,0.0,0.34106640240763725,0.0412382762971965,0.26913610432718743,0.032963919757285645,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2956334447581319,0.031067389793786468,0.29074894802042156,0.020759035076338427,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.002776942628903413,0.000977362983639031,0.20305608991187224,0.016932144951760125,1.365815153894008e-23,7.554680429632255e-24,0.00018074200217366542,5.945931546389664e-05,0.006728421310414885,0.00016309396929761914,,,,0.0017985867446393758,0.0006771314754316643,0.7494544621813646,0.014073975311252462,1.1258589212096723e-29,1.1202829815281729e-29,0.00017282752484634064,6.051558988995322e-05,0.009667772980172738,0.0003365933686375298,0.35,1.6687859049967576e-12,0.26487264673311184,,,,,,,,,,1.0619204030347365e-16,0.0,1.0619204030347365e-16,Baseline
251,0,KL Min.,Llama-2-7B,0.0,10,4,1e-05,0.7210884353741497,0.0,0.0,0.4349906763685168,0.041399677197376625,0.2617152784357442,0.025520569860729942,0.0,0.0,0.0064127027378722565,0.0002339181952486158,0.0,0.0,0.36218937769584114,0.03760897932258034,0.29083467610785985,0.025978663260389213,0.0,0.0,0.006258826904751285,0.00021636495911737754,0.0026436092955700796,0.0009694422922757615,0.16529649909914673,0.014232403245655625,2.0717691189905218e-08,4.243911736311607e-09,0.0001605399819716452,5.611579487497281e-05,0.015128848568515704,0.00028565751096415286,,,,0.0017985867446393758,0.0006771314754316643,0.6458844539685303,0.015464652835565633,1.574877261188353e-14,1.5596305250912948e-14,0.00017282752484634064,6.051558988995322e-05,0.009667772980172738,0.0003365933686375298,0.25,0.00019096774001327207,0.37574750830564785,,,,,,,,,,1.2161555648200794e-08,0.0,1.2161555648200794e-08,Baseline
227,0,Pref. Opt.,Llama-2-7B,0.0,10,0,1e-05,0.8208616780045351,0.349,0.04658351177212351,0.5885007803449822,0.022577712211044373,0.4546292566307752,0.01255699842330583,0.0830920550038197,0.0118839011751025,0.9919856459330144,0.0030199106704405865,0.7905982905982906,0.03608780030180388,0.5409891780052368,0.030264113017313667,0.44226483755413365,0.018008410760658337,0.11507276198439438,0.008380926467572483,0.9506182440443186,0.0062716661445706935,0.06981717780134428,0.01275426151668499,0.4186071896064514,0.012160514176653668,0.820828101619634,0.004678683675417054,0.10807352309526223,0.015638901063356036,0.9955341619725498,0.0013901013983054322,,,,0.026014018621585054,0.007623378378633632,0.5910928697584164,0.011800412721158515,0.7964434666506258,0.00486268062395443,0.04845238095238095,0.009380518950538016,0.9988422337065748,0.0006519473829531803,0.3333333333333333,9.635514732762048e-15,0.24208194905869324,,,,,,,,,,3.709652809739326e-15,0.2993130236159747,3.709652809739326e-15,Baseline
228,0,Pref. Opt.,Llama-2-7B,0.0,10,1,1e-05,0.8208616780045351,0.06199999999999999,0.02244844448804103,0.5588210977107396,0.026504838209564417,0.43523624742701317,0.014965691060851188,0.015976190476190477,0.005593054745790017,0.9550454545454545,0.004578952934133342,0.5683760683760684,0.04502066445053389,0.5171882622076496,0.032152150021254766,0.4174215613161179,0.019627539086695137,0.07427948779250088,0.007357732864079222,0.9303106045050152,0.006551093439922881,0.5267046457647624,0.024252315341516235,0.4230950475153902,0.013692146569582135,0.8755592645553156,0.006378099711610758,0.5259870440049652,0.02354571402058457,0.9305328277239128,0.003774976282447627,,,,0.061277851058873074,0.011322138338496426,0.5758466442889885,0.012288658847664221,0.8217270434008185,0.0062655927416635565,0.0954214724397645,0.012061225527843649,0.9831560905491679,0.001957671930141306,0.3,4.738095282671061e-17,0.22046511627906976,,,,,,,,,,2.5881511422773775e-12,0.2838313927741802,2.5881511422773775e-12,Baseline
229,0,Pref. Opt.,Llama-2-7B,0.0,10,2,1e-05,0.8208616780045351,0.35733333333333334,0.04759084879353266,0.5526503168094474,0.028984945329307503,0.432423089003636,0.016968126698097875,0.07732421087684246,0.010949492158650851,0.9914490937488649,0.0028600194928174567,0.8219373219373219,0.03425908309815274,0.5120588272372638,0.03413604767313588,0.4143660071117207,0.021557008904321058,0.10318426428779445,0.006429412890896463,0.9293544474737612,0.006926052866525464,0.6918876122063032,0.021877922663137286,0.4506596788237609,0.014061372363312993,0.922050824252585,0.004567215577115462,0.7202470611987749,0.020433681366050577,0.9367089208028301,0.003780576309351217,,,,0.10374269568676657,0.012625188678157219,0.5454546541892364,0.012424495630398547,0.8456309344937554,0.005781223007970579,0.16647581665516448,0.011438035071703836,0.9922395302030801,0.001563280559848138,0.3333333333333333,7.009678794215372e-27,0.14248062015503876,,,,,,,,,,3.709652809739326e-15,0.5216721950958199,3.709652809739326e-15,Baseline
230,0,Pref. Opt.,Llama-2-7B,0.0,10,3,1e-05,0.8208616780045351,0.6146666666666666,0.04803272360043588,0.56132185750228,0.029788705099224072,0.4395546536808594,0.01780238662649257,0.13263268018531177,0.011022724913657422,0.9573988051131264,0.004703249107181397,0.8133903133903134,0.03493560388692606,0.5194528571782538,0.03454344919906302,0.4202118466684467,0.022004658342985147,0.10268375110813233,0.0065937428964228445,0.9200466536369167,0.006416712917155158,0.7625219857942667,0.01961085128404821,0.46100548756873716,0.01412034058667831,0.9357573116100575,0.003974188778366592,0.7824214660978699,0.01899557984575811,0.9259736441010563,0.003300053055703975,,,,0.06082510876354154,0.01144722200523549,0.5398006093087906,0.012625245614030311,0.8276419618300567,0.005944180966896035,0.08878749953749954,0.011682944694107085,0.911264961040553,0.0009917005596740411,0.33666666666666667,2.1434549009040572e-30,0.11838316722037652,,,,,,,,,,1.8502009026131303e-15,0.5708133237545386,1.8502009026131303e-15,Baseline
231,0,Pref. Opt.,Llama-2-7B,0.0,10,4,1e-05,0.8208616780045351,0.7313333333333333,0.04383885397504271,0.548320668433842,0.030488207658856525,0.43134591555313534,0.01846937345275885,0.148571563369551,0.010099760158172844,0.9707312395134835,0.004437173753020781,0.8561253561253561,0.031206459854159443,0.5118411253605883,0.03497879989530106,0.4164981001571263,0.022367768113513594,0.10838170717274538,0.00650043780546534,0.9219647745882619,0.006723696287322877,0.8138883783454983,0.01733657859425645,0.46183777595680436,0.014262868141484798,0.9446237675099981,0.003922565985447677,0.8318081450030967,0.01643946160397134,0.9270252494915902,0.003513479566589719,,,,0.04825888777440945,0.010231256776888057,0.5416982368540082,0.01284763333924324,0.8138999759523842,0.006169204440312425,0.08293497632971317,0.010706712036329922,0.9661635410942686,0.00260308978036796,0.3233333333333333,8.097382402130872e-30,0.12223698781838317,,,,,,,,,,2.858944731506222e-14,0.5812169529105777,2.858944731506222e-14,Baseline
253,0,Retain Model,Llama-2-7B,0.0,10,-1,1e-05,0.0,0.9279999999999999,0.024166091947189144,0.5947988854000636,0.033605223087774444,0.44980359540221637,0.02234923922112359,0.17411979111359915,0.006795951960130148,0.9719513071314938,0.005006360676130846,0.9088319088319089,0.025711199561382098,0.5459951074987635,0.03578297274991433,0.42124277033515506,0.02677498414458228,0.1031676073808966,0.005784872630771222,0.9140786084176485,0.0071238723761199345,0.9810537904728946,0.005545055365200779,0.47393877357373354,0.014249411674370585,0.9893784260662525,0.0007522381003083327,0.9838158979130903,0.004790903384963573,0.9246379981408999,0.0032924872531743506,,,,0.39422382472862666,0.010360220997732228,0.6711315665909536,0.012556145190908048,0.1460792032498096,0.007758181724581559,0.3957377806201532,0.010624726080647672,0.904281838859758,0.004018981875738075,0.0,1.0,0.0,,,,,,,,,,1.0,0.6237955595741369,1.0,Baseline
232,0,Grad. Ascent,Llama-2-7B,0.01,10,0,1e-05,0.8888888888888888,0.945,0.02105188395801711,0.6187487585193752,0.029973667964308982,0.47501447977915695,0.02007173148500605,0.18622038513601982,0.006581976367816515,0.9708367168970651,0.005188968582688854,0.8917378917378919,0.02724332888874447,0.595287258859532,0.034329055471394955,0.4748396945412093,0.02928818019336817,0.10978678007162364,0.006239153050772274,0.924176642795394,0.007383537167726773,0.7076934448033372,0.01578149214116453,0.4573690568506355,0.012538971655043264,0.871716620502705,0.004754612902815609,0.7192132372798172,0.016004285369496184,0.9058151344389838,0.004756613728251303,,,,0.5984424176148694,0.015336641613849764,0.55819401989677,0.012319882647525952,0.7875850838858941,0.008101292071784954,0.6259319067792977,0.015904413181527088,0.9016045590506145,0.00475421784960993,0.3466666666666667,1.0956261248491814e-16,0.22376522702104099,,,,,,,,,,2.1942743021891237e-16,0.6235525384005506,2.1942743021891237e-16,Baseline
233,0,Grad. Ascent,Llama-2-7B,0.01,10,1,1e-05,0.8888888888888888,0.0,0.0,0.19017189691853528,0.025749612454904293,0.26578291541651766,0.014908871829083703,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.16566073502922174,0.020870095070104136,0.2681023967225485,0.013047213862247763,0.0,0.0,0.011758684425253466,0.0001587906083539704,0.0,0.0,0.03235800970366027,0.0027224033868851636,9.366409784245131e-05,1.1151789076700771e-06,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9459451648127121,0.002712272355774447,9.271086699894434e-05,1.2201016457170914e-06,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5333333333333333,1.4973669324808988e-17,0.21599114064230343,,,,,,,,,,2.0885153422421613e-39,0.0,2.0885153422421613e-39,Baseline
234,0,Grad. Ascent,Llama-2-7B,0.01,10,2,1e-05,0.888888888888889,0.0,0.0,0.13636626883647138,0.018218438084703144,0.2548431626364454,0.009154751896633815,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.1314631652917845,0.018396323316543572,0.2551124401152376,0.009872690261049196,0.0,0.0,0.014051087744462232,0.0007150453334525698,0.0,0.0,0.025344187064717336,0.002152979317936579,6.263111280117667e-05,5.858673320057006e-07,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9501198892873849,0.0027787470753550198,5.5653963975915514e-05,9.966222187258457e-07,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5533333333333333,8.607191677387933e-18,0.21386489479512735,,,,,,,,,,1.4228925129969089e-42,0.0,1.4228925129969089e-42,Baseline
235,0,Grad. Ascent,Llama-2-7B,0.01,10,3,1e-05,0.8888888888888888,0.0,0.0,0.5552698981078624,0.04236402576093415,0.42725355015714167,0.04177364286477556,0.0,0.0,0.035205880111707506,0.0005093501879041153,0.0,0.0,0.5218883184769632,0.03823529418057141,0.43497967266153775,0.03877145287550543,0.0,0.0,0.028361574978966436,0.0005039275343647508,0.0,0.0,0.24654598165895852,0.019055973974939425,5.863093817103905e-23,2.4758018480035482e-23,0.0,0.0,0.03826366293740615,0.0002439897706441564,,,,0.0,0.0,0.43957046500086866,0.017909319836362247,6.459840309107607e-18,3.498821365726045e-18,0.0,0.0,0.038111203951432915,0.0002974244900519239,0.31333333333333335,0.03300651266258098,0.4289922480620155,,,,,,,,,,2.0608068972918464e-13,0.0,2.0608068972918464e-13,Baseline
236,0,Grad. Ascent,Llama-2-7B,0.01,10,4,1e-05,0.8888888888888888,0.0,0.0,0.504849139257392,0.04527589534556345,0.31376372681284725,0.039914232727733125,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.48067544880832946,0.04225244571720282,0.3726199292098841,0.04128048893549813,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.1942667381581949,0.016367476156085616,1.3060374368858345e-50,9.625248255663002e-51,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.5107680628113074,0.016206167692476032,9.991660241684096e-51,7.823474588255901e-51,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.26666666666666666,0.058676762413160534,0.4370321151716501,,,,,,,,,,8.690124079460009e-10,0.0,8.690124079460009e-10,Baseline
212,0,Grad. Diff.,Llama-2-7B,0.01,10,0,1e-05,0.8208616780045351,0.9329999999999999,0.02378587136868601,0.6044341571342846,0.03295403844927024,0.46839407555481827,0.022090022496097518,0.1796095700824803,0.00715842576061889,0.9731410232901755,0.005123464894959443,0.8703703703703705,0.03001669034410197,0.5754631800007903,0.03523088213461695,0.452703619302839,0.03157332063030803,0.10250601652761857,0.0063887216818942945,0.9168854619122493,0.0068838846483578854,0.9240141808908553,0.010863017541088014,0.47780056612410154,0.013769749526114948,0.9645564544738333,0.0034398398192402677,0.9239827454191956,0.011024987548198488,0.9211929721771196,0.003438739275478625,,,,0.7628701916181125,0.016218603493237366,0.5214295985678726,0.012881445771069767,0.8830187610028744,0.007961448278010633,0.7761175244208777,0.01570711610614146,0.9164478433442056,0.003852978249210618,0.38,6.200544599492621e-21,0.18737541528239202,,,,,,,,,,1.096624314778916e-19,0.6352017316602937,1.096624314778916e-19,Baseline
213,0,Grad. Diff.,Llama-2-7B,0.01,10,1,1e-05,0.8208616780045351,0.5578333333333333,0.04730234142120421,0.597399076284999,0.03657226137799652,0.4372842183430803,0.026876328726907105,0.1062610336840886,0.01016555309682237,0.9210354922631803,0.014897400813727698,0.8347578347578347,0.03233952540308097,0.6076858141004515,0.03700330996462044,0.4548952485727426,0.037325885029387225,0.1081939624630904,0.006231085124890657,0.9330963922463212,0.007325057987164338,0.2820281513217174,0.010302055954323147,0.4720819075840925,0.017541910057826682,0.12861225661177608,0.010266014417502141,0.3130985505562747,0.01125824855355396,0.8151398983349183,0.011366509599946563,,,,0.22890351582075502,0.008735109251248518,0.5166760164328829,0.015120346406399494,0.02797749589839668,0.004167421910138012,0.2573758111316908,0.009842303873877361,0.6918125042115183,0.01652302094735894,0.20666666666666667,2.9465612052258827e-05,0.3608637873754153,,,,,,,,,,5.0749555303276145e-06,0.37136995954344854,5.0749555303276145e-06,Baseline
214,0,Grad. Diff.,Llama-2-7B,0.01,10,2,1e-05,0.8208616780045351,0.147,0.03327636546133754,0.7152883532832506,0.03462380372836556,0.5848631109936556,0.03369161222468478,0.01565450698611169,0.005738428795905251,0.11994744747643918,0.020166710875855913,0.5893162393162393,0.04339049368207918,0.6896381931427219,0.03443033329209034,0.5141956225115176,0.03477084707033165,0.04905851146179271,0.005739531894616003,0.42710824073297227,0.03218318019420482,0.07379010824554239,0.004775792187145151,0.4530032901235258,0.016115559800401306,0.01937100024945274,0.0030592455262335766,0.03778894668639581,0.0035858207485331348,0.1881169109791714,0.0098784532822644,,,,0.0026808296270702282,0.0009656265843762572,0.6751735614722527,0.015665786459479542,3.030613392968039e-08,2.46788550568028e-08,0.00042739613704646703,0.00013117332020543297,0.006863812247074542,0.00017425482045506463,0.24,7.617974544503307e-05,0.3682392026578073,,,,,,,,,,5.439158189288132e-08,0.10924283628908843,5.439158189288132e-08,Baseline
215,0,Grad. Diff.,Llama-2-7B,0.01,10,3,1e-05,0.8208616780045351,0.4771666666666667,0.04749898398110019,0.7279976089525759,0.03485902716106826,0.5954786744821959,0.027806472862790126,0.07511874282242166,0.009413410996968179,0.5351868789269048,0.03108794704562312,0.809116809116809,0.0350001378456764,0.6468973396585224,0.03524502096591484,0.4999016951021712,0.03475473510530182,0.05772005979528125,0.004674357775533508,0.6511698504635343,0.02360279378091694,0.3096741126054053,0.010965745275366099,0.4591469667354204,0.015298215551960445,0.26800962161977326,0.010061193205802456,0.22817042640332877,0.007961236148395,0.6918101315053312,0.012769405424800916,,,,0.0021308296270702285,0.0007583302993385423,0.6305831982904452,0.016256156952678267,1.5056979096577731e-09,1.5042876151617233e-09,0.0003659630516579701,0.00011402112121218639,0.006693008174996586,0.00010315359489165173,0.2633333333333333,0.00011353306693858288,0.37145071982281286,,,,,,,,,,1.4942294797261553e-09,0.47196606929168716,1.4942294797261553e-09,Baseline
216,0,Grad. Diff.,Llama-2-7B,0.01,10,4,1e-05,0.8208616780045351,0.469,0.04751160293043248,0.7453841996446183,0.03327244376567869,0.5984568053310647,0.028215022482907977,0.07322183830081555,0.009504423766840535,0.4764790847453708,0.03451644085805309,0.8148148148148149,0.034321831060000335,0.6384253136147289,0.03511364017528178,0.4942245042890296,0.03471004999289998,0.059527793303241545,0.005028303362985733,0.6364918224421716,0.02581335354702919,0.35577236819922264,0.01162188868264195,0.46303656688553985,0.015838594293178326,0.38230518866783353,0.011643398799723774,0.2717274228601494,0.00978831627147335,0.736477445898485,0.013017131500073496,,,,0.0009965703678109694,0.0004834320048865688,0.6212948598570626,0.01634484834968054,6.127305690859893e-17,6.127305690859798e-17,0.0002410065944908113,8.079507483856646e-05,0.006396075534510508,4.5224150597273095e-05,0.24,0.0002507290106945618,0.37805094130675526,,,,,,,,,,5.439158189288132e-08,0.5128817919151487,5.439158189288132e-08,Baseline
242,0,KL Min.,Llama-2-7B,0.01,10,0,1e-05,0.7210884353741497,0.935,0.023066657908563763,0.6133718552480142,0.030304737158523016,0.4707526603209077,0.02034364972398901,0.18233871125434592,0.006957184240937999,0.9715017415276072,0.005089830573148239,0.8917378917378919,0.02724332888874447,0.5892156089778035,0.034574670045988284,0.4711302936223364,0.029618872617726534,0.10932319695344021,0.006239541420652388,0.923197957862593,0.0073699327347598875,0.7404904100854898,0.015514693358977427,0.4612156167410793,0.012596562072535716,0.8846310673739973,0.004609736816808154,0.7449481316280455,0.01575808557430623,0.9082767535040845,0.004804146250715584,,,,0.6133219133758041,0.015676852894546976,0.554160597170425,0.012345522594458872,0.7955417828268352,0.008328697839953632,0.6394357786029153,0.016100041853064496,0.9030354865198414,0.004630880510777976,0.3466666666666667,2.81005469471046e-17,0.21842746400885935,,,,,,,,,,2.1942743021891237e-16,0.624380140246935,2.1942743021891237e-16,Baseline
243,0,KL Min.,Llama-2-7B,0.01,10,1,1e-05,0.7210884353741497,0.0,0.0,0.238464139683634,0.029646684960232,0.2947606897739976,0.01761371602350222,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.18568873407805564,0.022450669084094,0.2798750675153093,0.014223067967476515,0.0,0.0,0.011561045688060979,3.304553176102566e-05,0.0,0.0,0.03213729194016307,0.0027016841484961023,9.35232698524954e-05,1.04836771141367e-06,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9464136463909026,0.0027191624904088275,9.336259214701391e-05,1.2385687783025482e-06,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5366666666666666,1.7685834282543408e-17,0.21663344407530455,,,,,,,,,,6.3425628614729406e-40,0.0,6.3425628614729406e-40,Baseline
244,0,KL Min.,Llama-2-7B,0.01,10,2,1e-05,0.7210884353741497,0.0,0.0,0.3936456744555,0.036225724289654475,0.350643514377555,0.025719581499431092,0.0,0.0,0.0530176844286043,0.00206604726826176,0.008547008547008548,0.006017541110844508,0.39845067794737415,0.03562419862189332,0.39062422906428024,0.030081709237959546,0.00013108292577090322,9.232299997272141e-05,0.0534071180420904,0.001991000156626323,0.00669847746329261,0.0015099302861793134,0.03193739158008537,0.0025645952566283733,8.76876952298274e-05,3.450073150153303e-06,0.005359026898977892,0.0018216827374817866,0.05866748730238602,0.0015286622464291207,,,,0.0003935185185185185,0.000278277323433988,0.943652872803225,0.0031943976121007267,4.5601102193978194e-05,1.2222629383366072e-06,0.0004166666666666667,0.0003433216436840604,0.02409383462708579,0.001189833762448708,0.5466666666666666,2.7624167927647097e-17,0.2183610188261351,,,,,,,,,,1.6793386636236394e-41,0.0,1.6793386636236394e-41,Baseline
245,0,KL Min.,Llama-2-7B,0.01,10,3,1e-05,0.7210884353741497,0.0,0.0,0.5321772302857819,0.04628307817096065,0.37136380861364576,0.04249495012316642,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.49960339782644414,0.04259371941976864,0.3528943720722415,0.041515594572534696,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.19925104640484928,0.016512874204130022,7.271822890267727e-42,6.008798109027888e-42,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.4848340319767787,0.01662580386285543,1.4490835507200026e-43,6.326111196707916e-44,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.2833333333333333,0.07374426336252336,0.44044296788482834,,,,,,,,,,5.191390347437448e-11,0.0,5.191390347437448e-11,Baseline
246,0,KL Min.,Llama-2-7B,0.01,10,4,1e-05,0.7210884353741497,0.0,0.0,0.49764014345177116,0.046819008985518336,0.2843347363522162,0.0389223340713372,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.48041996668378434,0.043065061323862515,0.35049865513981787,0.04060085348588588,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.17853184237378694,0.01569468977387372,4.715506085151057e-61,3.4800339636551286e-61,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.5101658771973803,0.016075024386026513,4.135968240130737e-62,1.950218244233312e-62,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.27,0.04165921875451196,0.4321594684385382,,,,,,,,,,5.01791941596116e-10,0.0,5.01791941596116e-10,Baseline
222,0,Pref. Opt.,Llama-2-7B,0.01,10,0,1e-05,0.8208616780045351,0.37233333333333335,0.047045253663423825,0.5747826704930294,0.022849857100624225,0.44880716453005887,0.012946559289478082,0.0868266814884462,0.011641537628901816,0.9906761221234907,0.003133851567632787,0.7905982905982906,0.03608780030180388,0.5303338377740737,0.03049196810163221,0.441235403554138,0.020838890900462138,0.11602329651697596,0.008335675165517583,0.9533925007479708,0.005961710553429642,0.07434148604521824,0.013029736462718338,0.42049895460469316,0.012108259682437481,0.8160908256066413,0.0046802303284757705,0.1080846924814316,0.015462345403289187,0.9936868898939095,0.0017010734139741486,,,,0.028277057994119316,0.007965292118686412,0.5930864113944144,0.011841383325235945,0.7879623341996244,0.004929242570194882,0.05130952380952381,0.00989755530564852,0.9990878477416626,0.0005683075029403473,0.3233333333333333,1.1492587912901443e-10,0.2853156146179402,,,,,,,,,,2.858944731506222e-14,0.30900825088215067,2.858944731506222e-14,Baseline
223,0,Pref. Opt.,Llama-2-7B,0.01,10,1,1e-05,0.8208616780045351,0.06199999999999999,0.02244844448804103,0.5503895341459227,0.027529462899201208,0.4349964050751066,0.015904772910049554,0.015654761904761904,0.0054748669113853345,0.9749580419580419,0.004196233231343522,0.5826210826210827,0.04481425552815165,0.49997377594307063,0.033465782078326387,0.4223165442478161,0.023514937537052354,0.07762327979059412,0.007440879266492267,0.937517938121109,0.006353850813122646,0.4824519774260634,0.025888887482011673,0.423752270204518,0.01371076968041236,0.8777211397151145,0.0062684662493249985,0.4844270433615668,0.024849760334561682,0.9465048078425086,0.0038751731913320474,,,,0.04980129955723792,0.009644963435539073,0.577464211841622,0.012441048291071273,0.8196756259521027,0.005915193258850345,0.0814799254101586,0.009659806188445728,0.9923094672768938,0.001879361634375717,0.29333333333333333,3.8567598361995186e-11,0.27986710963455147,,,,,,,,,,8.777796874606567e-12,0.28212189963542056,8.777796874606567e-12,Baseline
224,0,Pref. Opt.,Llama-2-7B,0.01,10,2,1e-05,0.8208616780045351,0.29133333333333333,0.04497316820711943,0.5475407125684502,0.028960406822220543,0.43162688013468,0.017564841653853462,0.06354737660000818,0.010517512451336818,0.9890253719587336,0.0032539991601581884,0.7934472934472935,0.036622385080893015,0.49517917025551594,0.03476778421692807,0.4190363239807755,0.025108561239523465,0.10279450910247236,0.006699578536422261,0.9411200021821509,0.006183728525904985,0.6828504177730427,0.022262394811320772,0.446238155102904,0.014013804475529752,0.915881406542059,0.004959978994002249,0.7057888619841924,0.021200239034826405,0.9356739696374873,0.003783871748562699,,,,0.10198269458313836,0.013198848712539986,0.5537857026468429,0.012784410679399106,0.8439264730253759,0.00542037471883288,0.16414206237232554,0.01258984275626304,0.9925432123462611,0.001605157679267544,0.32,1.9053339910013803e-14,0.24498338870431893,,,,,,,,,,5.563730678470443e-14,0.498915403815241,5.563730678470443e-14,Baseline
225,0,Pref. Opt.,Llama-2-7B,0.01,10,3,1e-05,0.8208616780045351,0.5846666666666667,0.04865744112548367,0.5400212376424239,0.029669493442535416,0.42687285242145395,0.018235922311307263,0.12580395335658495,0.011394220612185809,0.9523487188344092,0.00482466802738278,0.8390313390313391,0.03280675186486188,0.494666242119652,0.035019207228537734,0.41800922410335023,0.025548387823867254,0.10402994944230111,0.00629836411628782,0.919879552552894,0.0065607985781069465,0.774547418438988,0.02032252084544217,0.4551786813511306,0.013984467524247847,0.9368884339109415,0.00380461750460525,0.7862055863021258,0.019551089894799253,0.9258696608260304,0.0033838746869705393,,,,0.08397916876683457,0.013218894282992845,0.5477669687086352,0.012911001249366085,0.839265697788524,0.005362487991475072,0.12287430065061644,0.013592981224766003,0.9322256101284517,0.0023143343324680196,0.33666666666666667,1.4785541822365532e-15,0.23426356589147287,,,,,,,,,,1.8502009026131303e-15,0.5601941167018999,1.8502009026131303e-15,Baseline
226,0,Pref. Opt.,Llama-2-7B,0.01,10,4,1e-05,0.8208616780045351,0.6496666666666667,0.046780752802537996,0.5358880903702782,0.029964606084025514,0.4246043102292606,0.01854252964929691,0.13681449836712994,0.010917621513464672,0.9595042212267535,0.004761058803675313,0.8475783475783476,0.03202643916106147,0.49298545701035773,0.03514675499374474,0.4169656011526384,0.02575251703752355,0.1034027130556435,0.006253261375128168,0.9196098342140809,0.006836431821863783,0.8235664994975654,0.017909103149189986,0.4573389752774102,0.013994122673080957,0.9450982542002703,0.003475079029212093,0.8384847297951948,0.017008985636463032,0.926449565804033,0.0033328728586064175,,,,0.06266997094265174,0.011774912931646194,0.5469193067493091,0.012932046786020229,0.8366279850677959,0.005404314441473742,0.08752151211361738,0.011839264225030333,0.911084098266416,0.0009251037039683804,0.34,8.6557217874757e-16,0.23207087486157255,,,,,,,,,,9.158954716915745e-16,0.5686880809848377,9.158954716915745e-16,Baseline
252,0,Retain Model,Llama-2-7B,0.01,10,-1,1e-05,0.0,0.9179999999999999,0.02587381969200809,0.5867992311686108,0.034162794309235134,0.4445423697965439,0.022638593539283134,0.1737265650586084,0.007311711447675104,0.9724468340248467,0.005002158226125613,0.9088319088319089,0.025711199561382098,0.5468444548453482,0.03562985978961973,0.41868767541998175,0.026761345244793615,0.10158596746207914,0.0057138691806951055,0.9139116515846981,0.007144110342348025,0.9822053126894559,0.005094663869762722,0.471867662555522,0.01432314572853207,0.9897651615521136,0.000750840077271583,0.9846385499993602,0.004445153516169368,0.9240800326247677,0.0033220938143150103,,,,0.3983512708804085,0.010367448527135203,0.6694346204288318,0.012469078839573993,0.14361598711400606,0.007708751753109203,0.4025678758675452,0.010442822696508139,0.906735889275128,0.003856405682599201,0.0,1.0,0.0,,,,,,,,,,1.0,0.6203436494291608,1.0,Baseline