diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2413 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8663169145584106,
-      "logits/rejected": -1.870638370513916,
-      "logps/chosen": -36.98221206665039,
-      "logps/rejected": -33.6473503112793,
-      "loss": 0.6788,
-      "rewards/accuracies": 0.5416666865348816,
-      "rewards/chosen": 0.01950961910188198,
-      "rewards/margins": 0.03332838416099548,
-      "rewards/rejected": -0.013818766921758652,
+      "logits/chosen": -1.866452693939209,
+      "logits/rejected": -1.8707809448242188,
+      "logps/chosen": -36.98366165161133,
+      "logps/rejected": -33.666290283203125,
+      "loss": 0.4914,
+      "rewards/accuracies": 0.5555555820465088,
+      "rewards/chosen": 0.01834939420223236,
+      "rewards/margins": 0.04732292518019676,
+      "rewards/rejected": -0.02897353284060955,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9978923797607422,
-      "logits/rejected": -2.0005345344543457,
-      "logps/chosen": -29.642324447631836,
-      "logps/rejected": -29.048343658447266,
-      "loss": 0.7013,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.00010492801811778918,
-      "rewards/margins": -0.012185259722173214,
-      "rewards/rejected": 0.012080332264304161,
+      "logits/chosen": -1.9971332550048828,
+      "logits/rejected": -1.999770164489746,
+      "logps/chosen": -29.6362247467041,
+      "logps/rejected": -29.044448852539062,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.004776014480739832,
+      "rewards/margins": -0.01042011845856905,
+      "rewards/rejected": 0.015196132473647594,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9209339618682861,
-      "logits/rejected": -1.9182507991790771,
-      "logps/chosen": -31.401519775390625,
-      "logps/rejected": -33.22309875488281,
-      "loss": 0.6891,
+      "logits/chosen": -1.9208379983901978,
+      "logits/rejected": -1.9181442260742188,
+      "logps/chosen": -31.425155639648438,
+      "logps/rejected": -33.237945556640625,
+      "loss": 0.4997,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.011660982854664326,
-      "rewards/margins": 0.015369392931461334,
-      "rewards/rejected": -0.003708411008119583,
+      "rewards/chosen": -0.007249836809933186,
+      "rewards/margins": 0.008341209962964058,
+      "rewards/rejected": -0.015591045841574669,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0180399417877197,
-      "logits/rejected": -2.009289264678955,
-      "logps/chosen": -32.559410095214844,
-      "logps/rejected": -32.52582550048828,
-      "loss": 0.6849,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.013918718323111534,
-      "rewards/margins": 0.02444135770201683,
-      "rewards/rejected": -0.010522643104195595,
+      "logits/chosen": -2.0180625915527344,
+      "logits/rejected": -2.009308338165283,
+      "logps/chosen": -32.559749603271484,
+      "logps/rejected": -32.524681091308594,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.013646525330841541,
+      "rewards/margins": 0.023250887170433998,
+      "rewards/rejected": -0.009604359045624733,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8629716634750366,
-      "logits/rejected": -1.8522107601165771,
-      "logps/chosen": -33.554229736328125,
-      "logps/rejected": -35.44757080078125,
-      "loss": 0.6987,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.002600290346890688,
-      "rewards/margins": -0.0012185067171230912,
-      "rewards/rejected": 0.003818795783445239,
+      "logits/chosen": -1.8639278411865234,
+      "logits/rejected": -1.8531490564346313,
+      "logps/chosen": -33.53788375854492,
+      "logps/rejected": -35.42852020263672,
+      "loss": 0.5015,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.015677569434046745,
+      "rewards/margins": -0.0033827773295342922,
+      "rewards/rejected": 0.019060343503952026,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9408414363861084,
-      "logits/rejected": -1.9427950382232666,
-      "logps/chosen": -32.56097412109375,
-      "logps/rejected": -33.213417053222656,
-      "loss": 0.6725,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.03136637061834335,
-      "rewards/margins": 0.06499636918306351,
-      "rewards/rejected": -0.03362999111413956,
+      "logits/chosen": -1.9435577392578125,
+      "logits/rejected": -1.9454940557479858,
+      "logps/chosen": -32.49355697631836,
+      "logps/rejected": -33.1856575012207,
+      "loss": 0.4817,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08530057966709137,
+      "rewards/margins": 0.09672373533248901,
+      "rewards/rejected": -0.011423162184655666,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.072221517562866,
-      "logits/rejected": -2.077198028564453,
-      "logps/chosen": -33.974578857421875,
-      "logps/rejected": -36.629173278808594,
-      "loss": 0.6793,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.011124782264232635,
-      "rewards/margins": 0.059367585927248,
-      "rewards/rejected": -0.04824279993772507,
+      "logits/chosen": -2.075368881225586,
+      "logits/rejected": -2.0803427696228027,
+      "logps/chosen": -33.89010238647461,
+      "logps/rejected": -36.549957275390625,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07870620489120483,
+      "rewards/margins": 0.06357409805059433,
+      "rewards/rejected": 0.015132094733417034,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9338233470916748,
-      "logits/rejected": -1.9369605779647827,
-      "logps/chosen": -34.30416488647461,
-      "logps/rejected": -34.634437561035156,
-      "loss": 0.6423,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.09891629219055176,
-      "rewards/margins": 0.132537841796875,
-      "rewards/rejected": -0.03362155705690384,
+      "logits/chosen": -1.9372116327285767,
+      "logits/rejected": -1.9403302669525146,
+      "logps/chosen": -34.205604553222656,
+      "logps/rejected": -34.49314880371094,
+      "loss": 0.4788,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.17776472866535187,
+      "rewards/margins": 0.09835849702358246,
+      "rewards/rejected": 0.0794062465429306,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9418385028839111,
-      "logits/rejected": -1.9463545083999634,
-      "logps/chosen": -32.39947509765625,
-      "logps/rejected": -32.35419464111328,
-      "loss": 0.6785,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.05435952544212341,
-      "rewards/margins": 0.05058818310499191,
-      "rewards/rejected": 0.0037713423371315002,
+      "logits/chosen": -1.945416808128357,
+      "logits/rejected": -1.9499365091323853,
+      "logps/chosen": -32.26136016845703,
+      "logps/rejected": -32.26020050048828,
+      "loss": 0.4788,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.16484788060188293,
+      "rewards/margins": 0.08588258177042007,
+      "rewards/rejected": 0.07896529138088226,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.039783000946045,
-      "logits/rejected": -2.037789821624756,
-      "logps/chosen": -32.164188385009766,
-      "logps/rejected": -31.309520721435547,
-      "loss": 0.6538,
+      "logits/chosen": -2.043586254119873,
+      "logits/rejected": -2.0416104793548584,
+      "logps/chosen": -31.968154907226562,
+      "logps/rejected": -31.14252281188965,
+      "loss": 0.4749,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.061319977045059204,
-      "rewards/margins": 0.09596750140190125,
-      "rewards/rejected": -0.03464752808213234,
+      "rewards/chosen": 0.21814338862895966,
+      "rewards/margins": 0.11919467151165009,
+      "rewards/rejected": 0.09894871711730957,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2338831424713135,
-      "eval_logits/rejected": -2.229041814804077,
-      "eval_logps/chosen": -34.03020095825195,
-      "eval_logps/rejected": -37.52727127075195,
-      "eval_loss": 0.6954607963562012,
-      "eval_rewards/accuracies": 0.510797381401062,
-      "eval_rewards/chosen": 0.0034842013847082853,
-      "eval_rewards/margins": 0.012004716321825981,
-      "eval_rewards/rejected": -0.008520514704287052,
-      "eval_runtime": 146.115,
-      "eval_samples_per_second": 2.347,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.2375268936157227,
+      "eval_logits/rejected": -2.2326817512512207,
+      "eval_logps/chosen": -33.851219177246094,
+      "eval_logps/rejected": -37.3537483215332,
+      "eval_loss": 0.49629145860671997,
+      "eval_rewards/accuracies": 0.5336378812789917,
+      "eval_rewards/chosen": 0.14666710793972015,
+      "eval_rewards/margins": 0.016369162127375603,
+      "eval_rewards/rejected": 0.1302979290485382,
+      "eval_runtime": 145.7938,
+      "eval_samples_per_second": 2.353,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9946181774139404,
-      "logits/rejected": -1.992236852645874,
-      "logps/chosen": -33.117286682128906,
-      "logps/rejected": -34.00868225097656,
-      "loss": 0.6852,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.10122231394052505,
-      "rewards/margins": 0.0846698135137558,
-      "rewards/rejected": 0.016552483662962914,
+      "logits/chosen": -1.998853087425232,
+      "logits/rejected": -1.9964749813079834,
+      "logps/chosen": -32.937965393066406,
+      "logps/rejected": -33.846004486083984,
+      "loss": 0.474,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.2446821928024292,
+      "rewards/margins": 0.09798828512430191,
+      "rewards/rejected": 0.1466939002275467,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.006875514984131,
-      "logits/rejected": -1.9985148906707764,
-      "logps/chosen": -32.336421966552734,
-      "logps/rejected": -32.137081146240234,
-      "loss": 0.675,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.08678452670574188,
-      "rewards/margins": 0.06724556535482407,
-      "rewards/rejected": 0.019538963213562965,
+      "logits/chosen": -2.0093610286712646,
+      "logits/rejected": -2.0010437965393066,
+      "logps/chosen": -32.1511116027832,
+      "logps/rejected": -31.939533233642578,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.2350343018770218,
+      "rewards/margins": 0.057456426322460175,
+      "rewards/rejected": 0.17757786810398102,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0351502895355225,
-      "logits/rejected": -2.02717661857605,
-      "logps/chosen": -30.30923843383789,
-      "logps/rejected": -32.08501434326172,
-      "loss": 0.6398,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13318516314029694,
-      "rewards/margins": 0.15878939628601074,
-      "rewards/rejected": -0.025604233145713806,
+      "logits/chosen": -2.0374419689178467,
+      "logits/rejected": -2.029500961303711,
+      "logps/chosen": -30.157424926757812,
+      "logps/rejected": -31.90500831604004,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.2546332776546478,
+      "rewards/margins": 0.1362351030111313,
+      "rewards/rejected": 0.11839816719293594,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.966170072555542,
-      "logits/rejected": -1.9764087200164795,
-      "logps/chosen": -31.215194702148438,
-      "logps/rejected": -32.55674743652344,
-      "loss": 0.627,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.16357474029064178,
-      "rewards/margins": 0.18106886744499207,
-      "rewards/rejected": -0.017494117841124535,
+      "logits/chosen": -1.9675471782684326,
+      "logits/rejected": -1.9777450561523438,
+      "logps/chosen": -31.059162139892578,
+      "logps/rejected": -32.384300231933594,
+      "loss": 0.4616,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.28839823603630066,
+      "rewards/margins": 0.16793587803840637,
+      "rewards/rejected": 0.12046238034963608,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8775428533554077,
-      "logits/rejected": -1.8786998987197876,
-      "logps/chosen": -33.92055130004883,
-      "logps/rejected": -34.77721405029297,
-      "loss": 0.6168,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.21771302819252014,
-      "rewards/margins": 0.23367898166179657,
-      "rewards/rejected": -0.015965968370437622,
+      "logits/chosen": -1.8793624639511108,
+      "logits/rejected": -1.8805418014526367,
+      "logps/chosen": -33.646915435791016,
+      "logps/rejected": -34.56272888183594,
+      "loss": 0.4385,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.4366222321987152,
+      "rewards/margins": 0.2809991240501404,
+      "rewards/rejected": 0.15562310814857483,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9291296005249023,
-      "logits/rejected": -1.9257177114486694,
-      "logps/chosen": -36.01557922363281,
-      "logps/rejected": -32.72490692138672,
-      "loss": 0.6444,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12488000094890594,
-      "rewards/margins": 0.12660440802574158,
-      "rewards/rejected": -0.0017244067275896668,
+      "logits/chosen": -1.9307949542999268,
+      "logits/rejected": -1.9274256229400635,
+      "logps/chosen": -35.765926361083984,
+      "logps/rejected": -32.48193359375,
+      "loss": 0.4685,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.32460084557533264,
+      "rewards/margins": 0.13194666802883148,
+      "rewards/rejected": 0.19265416264533997,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0295331478118896,
-      "logits/rejected": -2.0221762657165527,
-      "logps/chosen": -33.48248291015625,
-      "logps/rejected": -31.408077239990234,
-      "loss": 0.5816,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.25230517983436584,
-      "rewards/margins": 0.2972865700721741,
-      "rewards/rejected": -0.04498137906193733,
+      "logits/chosen": -2.0318870544433594,
+      "logits/rejected": -2.0245723724365234,
+      "logps/chosen": -33.25593566894531,
+      "logps/rejected": -31.189266204833984,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4335424304008484,
+      "rewards/margins": 0.3034728467464447,
+      "rewards/rejected": 0.1300695389509201,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.036190986633301,
-      "logits/rejected": -2.0414373874664307,
-      "logps/chosen": -32.22594451904297,
-      "logps/rejected": -32.46149444580078,
-      "loss": 0.5993,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2550733685493469,
-      "rewards/margins": 0.2361568957567215,
-      "rewards/rejected": 0.018916476517915726,
+      "logits/chosen": -2.0387840270996094,
+      "logits/rejected": -2.0439517498016357,
+      "logps/chosen": -31.977941513061523,
+      "logps/rejected": -32.183441162109375,
+      "loss": 0.4499,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.4534761905670166,
+      "rewards/margins": 0.2121172845363617,
+      "rewards/rejected": 0.2413589060306549,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0371222496032715,
-      "logits/rejected": -2.0343565940856934,
-      "logps/chosen": -31.28468894958496,
-      "logps/rejected": -31.336734771728516,
-      "loss": 0.6307,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.16341081261634827,
-      "rewards/margins": 0.17822694778442383,
-      "rewards/rejected": -0.014816122129559517,
+      "logits/chosen": -2.039411783218384,
+      "logits/rejected": -2.036684513092041,
+      "logps/chosen": -31.094594955444336,
+      "logps/rejected": -31.10666275024414,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3154846429824829,
+      "rewards/margins": 0.1462438404560089,
+      "rewards/rejected": 0.169240802526474,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9067039489746094,
-      "logits/rejected": -1.91135573387146,
-      "logps/chosen": -31.312374114990234,
-      "logps/rejected": -32.82074737548828,
-      "loss": 0.6015,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.2351258099079132,
-      "rewards/margins": 0.25897616147994995,
-      "rewards/rejected": -0.0238503310829401,
+      "logits/chosen": -1.909691572189331,
+      "logits/rejected": -1.914358139038086,
+      "logps/chosen": -31.08693504333496,
+      "logps/rejected": -32.61079406738281,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.41547876596450806,
+      "rewards/margins": 0.27136775851249695,
+      "rewards/rejected": 0.14411096274852753,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.232191324234009,
-      "eval_logits/rejected": -2.22735333442688,
-      "eval_logps/chosen": -34.060691833496094,
-      "eval_logps/rejected": -37.56568908691406,
-      "eval_loss": 0.695566713809967,
-      "eval_rewards/accuracies": 0.52491694688797,
-      "eval_rewards/chosen": -0.020911961793899536,
-      "eval_rewards/margins": 0.01834380254149437,
-      "eval_rewards/rejected": -0.039255764335393906,
-      "eval_runtime": 145.8849,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.233895778656006,
+      "eval_logits/rejected": -2.2290823459625244,
+      "eval_logps/chosen": -33.78962326049805,
+      "eval_logps/rejected": -37.29551696777344,
+      "eval_loss": 0.4955826997756958,
+      "eval_rewards/accuracies": 0.5485880374908447,
+      "eval_rewards/chosen": 0.19594170153141022,
+      "eval_rewards/margins": 0.019058095291256905,
+      "eval_rewards/rejected": 0.17688362300395966,
+      "eval_runtime": 145.7262,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0197176933288574,
-      "logits/rejected": -2.0303761959075928,
-      "logps/chosen": -31.751026153564453,
-      "logps/rejected": -33.96234893798828,
-      "loss": 0.5969,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.17622438073158264,
-      "rewards/margins": 0.25594404339790344,
-      "rewards/rejected": -0.0797196701169014,
+      "logits/chosen": -2.0220179557800293,
+      "logits/rejected": -2.0326101779937744,
+      "logps/chosen": -31.51509666442871,
+      "logps/rejected": -33.655662536621094,
+      "loss": 0.4554,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.36496439576148987,
+      "rewards/margins": 0.19933317601680756,
+      "rewards/rejected": 0.16563120484352112,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9122480154037476,
-      "logits/rejected": -1.9270412921905518,
-      "logps/chosen": -29.86123275756836,
-      "logps/rejected": -31.612594604492188,
-      "loss": 0.6025,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.20024582743644714,
-      "rewards/margins": 0.24377915263175964,
-      "rewards/rejected": -0.04353334754705429,
+      "logits/chosen": -1.9142780303955078,
+      "logits/rejected": -1.9289791584014893,
+      "logps/chosen": -29.601001739501953,
+      "logps/rejected": -31.418704986572266,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.4084321856498718,
+      "rewards/margins": 0.2968555688858032,
+      "rewards/rejected": 0.11157669872045517,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9689687490463257,
-      "logits/rejected": -1.9729585647583008,
-      "logps/chosen": -33.11440658569336,
-      "logps/rejected": -31.650421142578125,
-      "loss": 0.5809,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.2424498349428177,
-      "rewards/margins": 0.3233444094657898,
-      "rewards/rejected": -0.08089461922645569,
+      "logits/chosen": -1.9722967147827148,
+      "logits/rejected": -1.9762967824935913,
+      "logps/chosen": -32.825157165527344,
+      "logps/rejected": -31.445331573486328,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.47385087609291077,
+      "rewards/margins": 0.39067354798316956,
+      "rewards/rejected": 0.0831773579120636,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9670231342315674,
-      "logits/rejected": -1.9451711177825928,
-      "logps/chosen": -33.830162048339844,
-      "logps/rejected": -35.1173095703125,
-      "loss": 0.558,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2588713765144348,
-      "rewards/margins": 0.3920826315879822,
-      "rewards/rejected": -0.13321125507354736,
+      "logits/chosen": -1.9708006381988525,
+      "logits/rejected": -1.9490963220596313,
+      "logps/chosen": -33.599205017089844,
+      "logps/rejected": -34.856910705566406,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4436335563659668,
+      "rewards/margins": 0.36852845549583435,
+      "rewards/rejected": 0.07510510087013245,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0084152221679688,
-      "logits/rejected": -2.005080223083496,
-      "logps/chosen": -32.70518493652344,
-      "logps/rejected": -36.280517578125,
-      "loss": 0.6104,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.1759072244167328,
-      "rewards/margins": 0.22038432955741882,
-      "rewards/rejected": -0.044477105140686035,
+      "logits/chosen": -2.011307954788208,
+      "logits/rejected": -2.008012533187866,
+      "logps/chosen": -32.494956970214844,
+      "logps/rejected": -35.98812484741211,
+      "loss": 0.4641,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.3440879285335541,
+      "rewards/margins": 0.15464913845062256,
+      "rewards/rejected": 0.18943879008293152,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8755052089691162,
-      "logits/rejected": -1.8730967044830322,
-      "logps/chosen": -33.984092712402344,
-      "logps/rejected": -35.538455963134766,
-      "loss": 0.622,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.16344432532787323,
-      "rewards/margins": 0.19051328301429749,
-      "rewards/rejected": -0.02706894651055336,
+      "logits/chosen": -1.8780018091201782,
+      "logits/rejected": -1.8755817413330078,
+      "logps/chosen": -33.72126388549805,
+      "logps/rejected": -35.290069580078125,
+      "loss": 0.4544,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.37370795011520386,
+      "rewards/margins": 0.20206721127033234,
+      "rewards/rejected": 0.17164072394371033,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8608484268188477,
-      "logits/rejected": -1.8584181070327759,
-      "logps/chosen": -34.17797088623047,
-      "logps/rejected": -31.830347061157227,
-      "loss": 0.6124,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1757020801305771,
-      "rewards/margins": 0.22291450202465057,
-      "rewards/rejected": -0.047212425619363785,
+      "logits/chosen": -1.8628206253051758,
+      "logits/rejected": -1.8603187799453735,
+      "logps/chosen": -33.929344177246094,
+      "logps/rejected": -31.61127281188965,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3746057152748108,
+      "rewards/margins": 0.246560737490654,
+      "rewards/rejected": 0.128044992685318,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9641139507293701,
-      "logits/rejected": -1.9535942077636719,
-      "logps/chosen": -35.01939010620117,
-      "logps/rejected": -31.871440887451172,
-      "loss": 0.5852,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.26980119943618774,
-      "rewards/margins": 0.29416200518608093,
-      "rewards/rejected": -0.024360809475183487,
+      "logits/chosen": -1.9666433334350586,
+      "logits/rejected": -1.9561984539031982,
+      "logps/chosen": -34.74022674560547,
+      "logps/rejected": -31.632495880126953,
+      "loss": 0.4239,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4931296706199646,
+      "rewards/margins": 0.3263343572616577,
+      "rewards/rejected": 0.16679534316062927,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0593204498291016,
-      "logits/rejected": -2.0443997383117676,
-      "logps/chosen": -30.722980499267578,
-      "logps/rejected": -32.61235809326172,
-      "loss": 0.6599,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.16091887652873993,
-      "rewards/margins": 0.12933868169784546,
-      "rewards/rejected": 0.031580209732055664,
+      "logits/chosen": -2.0619094371795654,
+      "logits/rejected": -2.0470869541168213,
+      "logps/chosen": -30.43317222595215,
+      "logps/rejected": -32.3420524597168,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.39276057481765747,
+      "rewards/margins": 0.1449340283870697,
+      "rewards/rejected": 0.24782654643058777,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.930450201034546,
-      "logits/rejected": -1.9279005527496338,
-      "logps/chosen": -32.415870666503906,
-      "logps/rejected": -30.882410049438477,
-      "loss": 0.5385,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4089924395084381,
-      "rewards/margins": 0.4614754319190979,
-      "rewards/rejected": -0.05248301103711128,
+      "logits/chosen": -1.9332126379013062,
+      "logits/rejected": -1.9307750463485718,
+      "logps/chosen": -32.100502014160156,
+      "logps/rejected": -30.6780948638916,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.6612862348556519,
+      "rewards/margins": 0.5503143072128296,
+      "rewards/rejected": 0.1109718531370163,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.229433536529541,
-      "eval_logits/rejected": -2.22458553314209,
-      "eval_logps/chosen": -34.090904235839844,
-      "eval_logps/rejected": -37.59726333618164,
-      "eval_loss": 0.69569993019104,
-      "eval_rewards/accuracies": 0.5398671627044678,
-      "eval_rewards/chosen": -0.04508008435368538,
-      "eval_rewards/margins": 0.019436603412032127,
-      "eval_rewards/rejected": -0.06451668590307236,
-      "eval_runtime": 145.8403,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.231203556060791,
+      "eval_logits/rejected": -2.226395606994629,
+      "eval_logps/chosen": -33.778892517089844,
+      "eval_logps/rejected": -37.28717803955078,
+      "eval_loss": 0.4949820339679718,
+      "eval_rewards/accuracies": 0.5245016813278198,
+      "eval_rewards/chosen": 0.204526886343956,
+      "eval_rewards/margins": 0.020971858873963356,
+      "eval_rewards/rejected": 0.1835550218820572,
+      "eval_runtime": 145.6189,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -1.9146509170532227,
-      "logits/rejected": -1.9113744497299194,
-      "logps/chosen": -31.32379150390625,
-      "logps/rejected": -33.805755615234375,
-      "loss": 0.5978,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.9173246622085571,
+      "logits/rejected": -1.9141733646392822,
+      "logps/chosen": -31.059444427490234,
+      "logps/rejected": -33.56504440307617,
+      "loss": 0.4356,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.2236466407775879,
-      "rewards/margins": 0.27452850341796875,
-      "rewards/rejected": -0.050881873816251755,
+      "rewards/chosen": 0.4351249635219574,
+      "rewards/margins": 0.293440043926239,
+      "rewards/rejected": 0.141684889793396,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9630565643310547,
-      "logits/rejected": -1.9508411884307861,
-      "logps/chosen": -34.32460403442383,
-      "logps/rejected": -33.67837905883789,
-      "loss": 0.5713,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.217172771692276,
-      "rewards/margins": 0.3288002014160156,
-      "rewards/rejected": -0.1116274744272232,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.9676717519760132,
+      "logits/rejected": -1.9555410146713257,
+      "logps/chosen": -34.03219985961914,
+      "logps/rejected": -33.442317962646484,
+      "loss": 0.417,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4510994851589203,
+      "rewards/margins": 0.37388378381729126,
+      "rewards/rejected": 0.07721573859453201,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.9971084594726562,
-      "logits/rejected": -1.995650053024292,
-      "logps/chosen": -33.14687728881836,
-      "logps/rejected": -32.54724884033203,
-      "loss": 0.5766,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2711312174797058,
-      "rewards/margins": 0.32526224851608276,
-      "rewards/rejected": -0.05413101240992546,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -2.002480983734131,
+      "logits/rejected": -2.0011117458343506,
+      "logps/chosen": -32.882102966308594,
+      "logps/rejected": -32.251502990722656,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.4829506278038025,
+      "rewards/margins": 0.30048781633377075,
+      "rewards/rejected": 0.18246281147003174,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -2.0822432041168213,
-      "logits/rejected": -2.066549301147461,
-      "logps/chosen": -33.73598098754883,
-      "logps/rejected": -33.096824645996094,
-      "loss": 0.572,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.35473155975341797,
-      "rewards/margins": 0.33776286244392395,
-      "rewards/rejected": 0.016968706622719765,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.0899081230163574,
+      "logits/rejected": -2.0742757320404053,
+      "logps/chosen": -33.487709045410156,
+      "logps/rejected": -32.8193359375,
+      "loss": 0.4286,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.5533460974693298,
+      "rewards/margins": 0.3143841624259949,
+      "rewards/rejected": 0.23896190524101257,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.9550272226333618,
-      "logits/rejected": -1.9541677236557007,
-      "logps/chosen": -32.79063415527344,
-      "logps/rejected": -32.5118293762207,
-      "loss": 0.5561,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3931701183319092,
-      "rewards/margins": 0.43051546812057495,
-      "rewards/rejected": -0.03734535723924637,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9617509841918945,
+      "logits/rejected": -1.9609159231185913,
+      "logps/chosen": -32.60249710083008,
+      "logps/rejected": -32.25555419921875,
+      "loss": 0.418,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.5436802506446838,
+      "rewards/margins": 0.37600547075271606,
+      "rewards/rejected": 0.16767482459545135,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -1.9071505069732666,
-      "logits/rejected": -1.917394995689392,
-      "logps/chosen": -31.765522003173828,
-      "logps/rejected": -35.38612365722656,
-      "loss": 0.5497,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.35133686661720276,
-      "rewards/margins": 0.4292295575141907,
-      "rewards/rejected": -0.07789268344640732,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9185073375701904,
+      "logits/rejected": -1.9287922382354736,
+      "logps/chosen": -31.57277488708496,
+      "logps/rejected": -35.039085388183594,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.505532443523407,
+      "rewards/margins": 0.305793821811676,
+      "rewards/rejected": 0.19973860681056976,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -2.042468547821045,
-      "logits/rejected": -2.0360734462738037,
-      "logps/chosen": -33.295223236083984,
-      "logps/rejected": -29.314971923828125,
-      "loss": 0.5615,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.28539514541625977,
-      "rewards/margins": 0.35623133182525635,
-      "rewards/rejected": -0.07083616405725479,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.0573890209198,
+      "logits/rejected": -2.0508790016174316,
+      "logps/chosen": -33.04835891723633,
+      "logps/rejected": -28.99324607849121,
+      "loss": 0.4308,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4828890860080719,
+      "rewards/margins": 0.2963466942310333,
+      "rewards/rejected": 0.18654237687587738,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -1.9002101421356201,
-      "logits/rejected": -1.9024240970611572,
-      "logps/chosen": -33.61799240112305,
-      "logps/rejected": -30.948089599609375,
-      "loss": 0.5044,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.49684303998947144,
-      "rewards/margins": 0.5572139024734497,
-      "rewards/rejected": -0.06037076562643051,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9167007207870483,
+      "logits/rejected": -1.9188674688339233,
+      "logps/chosen": -33.65839385986328,
+      "logps/rejected": -30.719829559326172,
+      "loss": 0.4247,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.46452397108078003,
+      "rewards/margins": 0.34228652715682983,
+      "rewards/rejected": 0.12223746627569199,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.9271265268325806,
-      "logits/rejected": -1.9258983135223389,
-      "logps/chosen": -33.7630500793457,
-      "logps/rejected": -35.974002838134766,
-      "loss": 0.4531,
-      "rewards/accuracies": 0.7958333492279053,
-      "rewards/chosen": 0.4973909258842468,
-      "rewards/margins": 0.7517890930175781,
-      "rewards/rejected": -0.2543981671333313,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.8622478246688843,
-      "logits/rejected": -1.8538604974746704,
-      "logps/chosen": -31.049022674560547,
-      "logps/rejected": -36.37419509887695,
-      "loss": 0.4017,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.5799586772918701,
-      "rewards/margins": 0.9268311262130737,
-      "rewards/rejected": -0.34687256813049316,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.2100114822387695,
-      "eval_logits/rejected": -2.205153226852417,
-      "eval_logps/chosen": -34.06473159790039,
-      "eval_logps/rejected": -37.634727478027344,
-      "eval_loss": 0.6775211691856384,
-      "eval_rewards/accuracies": 0.6004983186721802,
-      "eval_rewards/chosen": -0.024144915863871574,
-      "eval_rewards/margins": 0.07034175843000412,
-      "eval_rewards/rejected": -0.09448666870594025,
-      "eval_runtime": 146.4012,
-      "eval_samples_per_second": 2.343,
-      "eval_steps_per_second": 0.294,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -2.0343263149261475,
-      "logits/rejected": -2.037248134613037,
-      "logps/chosen": -32.27507019042969,
-      "logps/rejected": -35.283382415771484,
-      "loss": 0.3721,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.6230968236923218,
-      "rewards/margins": 1.0155293941497803,
-      "rewards/rejected": -0.3924326002597809,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.8720476627349854,
-      "logits/rejected": -1.8706938028335571,
-      "logps/chosen": -28.4278507232666,
-      "logps/rejected": -32.78199005126953,
-      "loss": 0.3947,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.5117626786231995,
-      "rewards/margins": 0.8832231760025024,
-      "rewards/rejected": -0.371460497379303,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.8284943103790283,
-      "logits/rejected": -1.821554183959961,
-      "logps/chosen": -33.15713119506836,
-      "logps/rejected": -34.481746673583984,
-      "loss": 0.3744,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.733561635017395,
-      "rewards/margins": 0.9227407574653625,
-      "rewards/rejected": -0.18917915225028992,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -1.998443841934204,
-      "logits/rejected": -1.993316888809204,
-      "logps/chosen": -30.946176528930664,
-      "logps/rejected": -32.4330940246582,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.669696033000946,
-      "rewards/margins": 1.0420799255371094,
-      "rewards/rejected": -0.3723840117454529,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -1.8814995288848877,
-      "logits/rejected": -1.8857187032699585,
-      "logps/chosen": -33.60875701904297,
-      "logps/rejected": -33.16690444946289,
-      "loss": 0.3134,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.7256715297698975,
-      "rewards/margins": 1.1882824897766113,
-      "rewards/rejected": -0.4626109004020691,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -1.8870223760604858,
-      "logits/rejected": -1.8813492059707642,
-      "logps/chosen": -34.445613861083984,
-      "logps/rejected": -35.65070343017578,
-      "loss": 0.3011,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.779128909111023,
-      "rewards/margins": 1.3404181003570557,
-      "rewards/rejected": -0.5612891316413879,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.9110435247421265,
-      "logits/rejected": -1.910775899887085,
-      "logps/chosen": -33.23371887207031,
-      "logps/rejected": -34.685794830322266,
-      "loss": 0.3634,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.747688353061676,
-      "rewards/margins": 1.1479377746582031,
-      "rewards/rejected": -0.40024933218955994,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -1.9573456048965454,
-      "logits/rejected": -1.938490867614746,
-      "logps/chosen": -33.21842575073242,
-      "logps/rejected": -33.71406555175781,
-      "loss": 0.3363,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.753872275352478,
-      "rewards/margins": 1.2691444158554077,
-      "rewards/rejected": -0.5152722001075745,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9228092432022095,
-      "logits/rejected": -1.9221388101577759,
-      "logps/chosen": -30.777042388916016,
-      "logps/rejected": -31.67429542541504,
-      "loss": 0.3684,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6674107313156128,
-      "rewards/margins": 1.0100107192993164,
-      "rewards/rejected": -0.34259992837905884,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.8837568759918213,
-      "logits/rejected": -1.8767935037612915,
-      "logps/chosen": -31.731576919555664,
-      "logps/rejected": -35.3281364440918,
-      "loss": 0.3216,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.7508381605148315,
-      "rewards/margins": 1.3119536638259888,
-      "rewards/rejected": -0.5611153841018677,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.170705556869507,
-      "eval_logits/rejected": -2.165867567062378,
-      "eval_logps/chosen": -34.18019104003906,
-      "eval_logps/rejected": -37.78302764892578,
-      "eval_loss": 0.6819708347320557,
-      "eval_rewards/accuracies": 0.5768272280693054,
-      "eval_rewards/chosen": -0.11651215702295303,
-      "eval_rewards/margins": 0.09661299735307693,
-      "eval_rewards/rejected": -0.21312516927719116,
-      "eval_runtime": 145.937,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.8785709142684937,
-      "logits/rejected": -1.8822866678237915,
-      "logps/chosen": -28.374109268188477,
-      "logps/rejected": -33.53699493408203,
-      "loss": 0.3449,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.6405723094940186,
-      "rewards/margins": 1.1827360391616821,
-      "rewards/rejected": -0.5421636700630188,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -1.8827375173568726,
-      "logits/rejected": -1.89346444606781,
-      "logps/chosen": -32.36341857910156,
-      "logps/rejected": -31.319293975830078,
-      "loss": 0.3592,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.7191308736801147,
-      "rewards/margins": 1.2532163858413696,
-      "rewards/rejected": -0.5340853929519653,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.829416036605835,
-      "logits/rejected": -1.8270241022109985,
-      "logps/chosen": -30.80521011352539,
-      "logps/rejected": -30.752178192138672,
-      "loss": 0.3535,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.6894973516464233,
-      "rewards/margins": 1.1498301029205322,
-      "rewards/rejected": -0.46033263206481934,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.9103914499282837,
-      "logits/rejected": -1.9033727645874023,
-      "logps/chosen": -33.29057312011719,
-      "logps/rejected": -33.72592544555664,
-      "loss": 0.33,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.7538042664527893,
-      "rewards/margins": 1.3205549716949463,
-      "rewards/rejected": -0.5667508244514465,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.8793199062347412,
-      "logits/rejected": -1.8857247829437256,
-      "logps/chosen": -30.955270767211914,
-      "logps/rejected": -33.297035217285156,
-      "loss": 0.3587,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.6383341550827026,
-      "rewards/margins": 1.0689740180969238,
-      "rewards/rejected": -0.43063968420028687,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -1.949784278869629,
-      "logits/rejected": -1.9266754388809204,
-      "logps/chosen": -30.55877685546875,
-      "logps/rejected": -33.56010818481445,
-      "loss": 0.3523,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.7030326128005981,
-      "rewards/margins": 1.1542710065841675,
-      "rewards/rejected": -0.45123839378356934,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.883561134338379,
-      "logits/rejected": -1.8786439895629883,
-      "logps/chosen": -31.494693756103516,
-      "logps/rejected": -32.63154220581055,
-      "loss": 0.3697,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.7376305460929871,
-      "rewards/margins": 1.1461317539215088,
-      "rewards/rejected": -0.4085012972354889,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -1.9235492944717407,
-      "logits/rejected": -1.9254448413848877,
-      "logps/chosen": -33.56511688232422,
-      "logps/rejected": -35.665016174316406,
-      "loss": 0.3133,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6485405564308167,
-      "rewards/margins": 1.339817762374878,
-      "rewards/rejected": -0.6912771463394165,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.9060910940170288,
-      "logits/rejected": -1.9054912328720093,
-      "logps/chosen": -29.636911392211914,
-      "logps/rejected": -32.14686965942383,
-      "loss": 0.332,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.7641160488128662,
-      "rewards/margins": 1.312302827835083,
-      "rewards/rejected": -0.5481868386268616,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.919725775718689,
-      "logits/rejected": -1.9174749851226807,
-      "logps/chosen": -31.991186141967773,
-      "logps/rejected": -32.90464782714844,
-      "loss": 0.336,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.8056806325912476,
-      "rewards/margins": 1.2894303798675537,
-      "rewards/rejected": -0.4837496280670166,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.160193920135498,
-      "eval_logits/rejected": -2.1554007530212402,
-      "eval_logps/chosen": -34.264404296875,
-      "eval_logps/rejected": -37.94013214111328,
-      "eval_loss": 0.6617584228515625,
-      "eval_rewards/accuracies": 0.6241694688796997,
-      "eval_rewards/chosen": -0.18387840688228607,
-      "eval_rewards/margins": 0.15492965281009674,
-      "eval_rewards/rejected": -0.3388080298900604,
-      "eval_runtime": 145.9206,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -1.9637705087661743,
-      "logits/rejected": -1.9703562259674072,
-      "logps/chosen": -31.264089584350586,
-      "logps/rejected": -32.738319396972656,
-      "loss": 0.3311,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.7786344289779663,
-      "rewards/margins": 1.3187401294708252,
-      "rewards/rejected": -0.5401058197021484,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -1.9196746349334717,
-      "logits/rejected": -1.9172741174697876,
-      "logps/chosen": -33.669281005859375,
-      "logps/rejected": -31.596511840820312,
-      "loss": 0.3157,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.869280993938446,
-      "rewards/margins": 1.3580834865570068,
-      "rewards/rejected": -0.48880258202552795,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.8648065328598022,
-      "logits/rejected": -1.8580557107925415,
-      "logps/chosen": -34.281700134277344,
-      "logps/rejected": -33.09290313720703,
-      "loss": 0.3058,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9460186958312988,
-      "rewards/margins": 1.5345759391784668,
-      "rewards/rejected": -0.5885573029518127,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.8759320974349976,
-      "logits/rejected": -1.8822145462036133,
-      "logps/chosen": -32.987449645996094,
-      "logps/rejected": -33.8790283203125,
-      "loss": 0.3408,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.7934818863868713,
-      "rewards/margins": 1.3044978380203247,
-      "rewards/rejected": -0.5110160708427429,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -1.9594967365264893,
-      "logits/rejected": -1.973659873008728,
-      "logps/chosen": -31.017492294311523,
-      "logps/rejected": -32.631839752197266,
-      "loss": 0.3573,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.6679788827896118,
-      "rewards/margins": 1.1753003597259521,
-      "rewards/rejected": -0.5073216557502747,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.8336044549942017,
-      "logits/rejected": -1.8305423259735107,
-      "logps/chosen": -32.8193473815918,
-      "logps/rejected": -35.608089447021484,
-      "loss": 0.2779,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.8828850984573364,
-      "rewards/margins": 1.6091547012329102,
-      "rewards/rejected": -0.726269543170929,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.9243332147598267,
-      "logits/rejected": -1.9244455099105835,
-      "logps/chosen": -30.38960838317871,
-      "logps/rejected": -34.67184066772461,
-      "loss": 0.3114,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.7167855501174927,
-      "rewards/margins": 1.3931505680084229,
-      "rewards/rejected": -0.676365077495575,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.8737255334854126,
-      "logits/rejected": -1.8705682754516602,
-      "logps/chosen": -29.965240478515625,
-      "logps/rejected": -33.58513259887695,
-      "loss": 0.3412,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.7256845235824585,
-      "rewards/margins": 1.339548945426941,
-      "rewards/rejected": -0.613864541053772,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.908151626586914,
-      "logits/rejected": -1.908117651939392,
-      "logps/chosen": -28.949966430664062,
-      "logps/rejected": -31.339040756225586,
-      "loss": 0.3023,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9705016016960144,
-      "rewards/margins": 1.5053366422653198,
-      "rewards/rejected": -0.5348349809646606,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.8213777542114258,
-      "logits/rejected": -1.824627161026001,
-      "logps/chosen": -33.35689163208008,
-      "logps/rejected": -32.15576934814453,
-      "loss": 0.3559,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.8972657322883606,
-      "rewards/margins": 1.4501286745071411,
-      "rewards/rejected": -0.5528628826141357,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.158259153366089,
-      "eval_logits/rejected": -2.1534817218780518,
-      "eval_logps/chosen": -34.35598373413086,
-      "eval_logps/rejected": -37.98072814941406,
-      "eval_loss": 0.6946965456008911,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": -0.2571484446525574,
-      "eval_rewards/margins": 0.1141389012336731,
-      "eval_rewards/rejected": -0.37128737568855286,
-      "eval_runtime": 145.9197,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -1.9044933319091797,
-      "logits/rejected": -1.8955074548721313,
-      "logps/chosen": -35.51199722290039,
-      "logps/rejected": -32.60811233520508,
-      "loss": 0.3093,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.8000162839889526,
-      "rewards/margins": 1.3618541955947876,
-      "rewards/rejected": -0.5618377923965454,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.9182393550872803,
-      "logits/rejected": -1.9193944931030273,
-      "logps/chosen": -35.33127212524414,
-      "logps/rejected": -33.76591110229492,
-      "loss": 0.3149,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.8832913637161255,
-      "rewards/margins": 1.435203194618225,
-      "rewards/rejected": -0.5519118309020996,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -1.9362226724624634,
-      "logits/rejected": -1.9337679147720337,
-      "logps/chosen": -31.4454345703125,
-      "logps/rejected": -33.67931365966797,
-      "loss": 0.3081,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.8943157196044922,
-      "rewards/margins": 1.4402475357055664,
-      "rewards/rejected": -0.5459317564964294,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.9176380634307861,
-      "logits/rejected": -1.922844648361206,
-      "logps/chosen": -30.585041046142578,
-      "logps/rejected": -33.932350158691406,
-      "loss": 0.2741,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9605496525764465,
-      "rewards/margins": 1.5825778245925903,
-      "rewards/rejected": -0.6220281720161438,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.864728569984436,
-      "logits/rejected": -1.8635470867156982,
-      "logps/chosen": -32.103065490722656,
-      "logps/rejected": -35.512298583984375,
-      "loss": 0.3075,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.8175305128097534,
-      "rewards/margins": 1.478601098060608,
-      "rewards/rejected": -0.6610704660415649,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.7420361042022705,
-      "logits/rejected": -1.7370550632476807,
-      "logps/chosen": -34.88098907470703,
-      "logps/rejected": -35.97365951538086,
-      "loss": 0.3032,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.8939619064331055,
-      "rewards/margins": 1.506162166595459,
-      "rewards/rejected": -0.6122000813484192,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.8814598321914673,
-      "logits/rejected": -1.8852901458740234,
-      "logps/chosen": -34.113304138183594,
-      "logps/rejected": -34.245601654052734,
-      "loss": 0.3271,
-      "rewards/accuracies": 0.85833340883255,
-      "rewards/chosen": 0.8802832365036011,
-      "rewards/margins": 1.4008936882019043,
-      "rewards/rejected": -0.5206103324890137,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.8257869482040405,
-      "logits/rejected": -1.8239891529083252,
-      "logps/chosen": -32.38594436645508,
-      "logps/rejected": -35.74359893798828,
-      "loss": 0.154,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.2125210762023926,
-      "rewards/margins": 2.2836754322052,
-      "rewards/rejected": -1.071154236793518,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.8724597692489624,
-      "logits/rejected": -1.8708446025848389,
-      "logps/chosen": -31.859329223632812,
-      "logps/rejected": -34.107154846191406,
-      "loss": 0.1817,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.2131247520446777,
-      "rewards/margins": 2.029592752456665,
-      "rewards/rejected": -0.8164681196212769,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.8005281686782837,
-      "logits/rejected": -1.7946115732192993,
-      "logps/chosen": -32.934303283691406,
-      "logps/rejected": -32.45271301269531,
-      "loss": 0.1978,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.1590179204940796,
-      "rewards/margins": 1.9497814178466797,
-      "rewards/rejected": -0.7907635569572449,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.154881000518799,
-      "eval_logits/rejected": -2.1501035690307617,
-      "eval_logps/chosen": -34.32504653930664,
-      "eval_logps/rejected": -37.97528076171875,
-      "eval_loss": 0.6837867498397827,
-      "eval_rewards/accuracies": 0.5834717750549316,
-      "eval_rewards/chosen": -0.2323925942182541,
-      "eval_rewards/margins": 0.1345335692167282,
-      "eval_rewards/rejected": -0.3669261634349823,
-      "eval_runtime": 145.9433,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.898033857345581,
-      "logits/rejected": -1.905503511428833,
-      "logps/chosen": -30.634164810180664,
-      "logps/rejected": -35.38495635986328,
-      "loss": 0.189,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.1952745914459229,
-      "rewards/margins": 2.0985639095306396,
-      "rewards/rejected": -0.9032894372940063,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.872579574584961,
-      "logits/rejected": -1.875286340713501,
-      "logps/chosen": -31.190942764282227,
-      "logps/rejected": -34.01335906982422,
-      "loss": 0.2433,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.104109525680542,
-      "rewards/margins": 1.7309566736221313,
-      "rewards/rejected": -0.6268469095230103,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.7723095417022705,
-      "logits/rejected": -1.7764465808868408,
-      "logps/chosen": -31.214824676513672,
-      "logps/rejected": -36.142818450927734,
-      "loss": 0.1808,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.1013058423995972,
-      "rewards/margins": 2.0874154567718506,
-      "rewards/rejected": -0.9861096143722534,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.8381569385528564,
-      "logits/rejected": -1.8343050479888916,
-      "logps/chosen": -31.591388702392578,
-      "logps/rejected": -33.85132598876953,
-      "loss": 0.2508,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.1018110513687134,
-      "rewards/margins": 1.7871520519256592,
-      "rewards/rejected": -0.6853411793708801,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.9189765453338623,
-      "logits/rejected": -1.9210529327392578,
-      "logps/chosen": -31.255529403686523,
-      "logps/rejected": -36.854835510253906,
-      "loss": 0.1574,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.2158453464508057,
-      "rewards/margins": 2.1701653003692627,
-      "rewards/rejected": -0.9543198347091675,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -1.7254867553710938,
-      "logits/rejected": -1.7197097539901733,
-      "logps/chosen": -34.86113357543945,
-      "logps/rejected": -33.97612762451172,
-      "loss": 0.2044,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.1364357471466064,
-      "rewards/margins": 2.003814697265625,
-      "rewards/rejected": -0.8673788905143738,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.7829675674438477,
-      "logits/rejected": -1.7830712795257568,
-      "logps/chosen": -34.304718017578125,
-      "logps/rejected": -35.99662780761719,
-      "loss": 0.181,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.2304695844650269,
-      "rewards/margins": 2.150757312774658,
-      "rewards/rejected": -0.9202877283096313,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.8317804336547852,
-      "logits/rejected": -1.8203016519546509,
-      "logps/chosen": -32.700950622558594,
-      "logps/rejected": -34.08301544189453,
-      "loss": 0.1692,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.1245036125183105,
-      "rewards/margins": 2.0532915592193604,
-      "rewards/rejected": -0.9287878274917603,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.9331659078598022,
-      "logits/rejected": -1.9377342462539673,
-      "logps/chosen": -31.932796478271484,
-      "logps/rejected": -33.6646728515625,
-      "loss": 0.1905,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1177093982696533,
-      "rewards/margins": 2.100264310836792,
-      "rewards/rejected": -0.9825549125671387,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.8520333766937256,
-      "logits/rejected": -1.844834327697754,
-      "logps/chosen": -33.29985427856445,
-      "logps/rejected": -33.635623931884766,
-      "loss": 0.1619,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.243377923965454,
-      "rewards/margins": 2.2301688194274902,
-      "rewards/rejected": -0.9867912530899048,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.1431386470794678,
-      "eval_logits/rejected": -2.138375997543335,
-      "eval_logps/chosen": -34.34238052368164,
-      "eval_logps/rejected": -38.03608703613281,
-      "eval_loss": 0.6788281202316284,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": -0.24626301229000092,
-      "eval_rewards/margins": 0.16930998861789703,
-      "eval_rewards/rejected": -0.41557300090789795,
-      "eval_runtime": 145.9104,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.8457978963851929,
-      "logits/rejected": -1.8462024927139282,
-      "logps/chosen": -29.82485008239746,
-      "logps/rejected": -36.06700897216797,
-      "loss": 0.1786,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0738495588302612,
-      "rewards/margins": 2.1757030487060547,
-      "rewards/rejected": -1.1018531322479248,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.8021042346954346,
-      "logits/rejected": -1.7940893173217773,
-      "logps/chosen": -30.521142959594727,
-      "logps/rejected": -34.798255920410156,
-      "loss": 0.1883,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0542433261871338,
-      "rewards/margins": 2.0357508659362793,
-      "rewards/rejected": -0.9815075993537903,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -1.9680095911026,
-      "logits/rejected": -1.968182921409607,
-      "logps/chosen": -28.589441299438477,
-      "logps/rejected": -33.940185546875,
-      "loss": 0.2295,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.1260759830474854,
-      "rewards/margins": 1.9863367080688477,
-      "rewards/rejected": -0.8602607846260071,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.847324013710022,
-      "logits/rejected": -1.857102394104004,
-      "logps/chosen": -32.901668548583984,
-      "logps/rejected": -34.3242073059082,
-      "loss": 0.1675,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.2187172174453735,
-      "rewards/margins": 2.2863640785217285,
-      "rewards/rejected": -1.0676472187042236,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.8901779651641846,
-      "logits/rejected": -1.8939307928085327,
-      "logps/chosen": -29.412405014038086,
-      "logps/rejected": -32.94640350341797,
-      "loss": 0.2048,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.9882534146308899,
-      "rewards/margins": 1.9565131664276123,
-      "rewards/rejected": -0.9682596921920776,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.8851401805877686,
-      "logits/rejected": -1.8811264038085938,
-      "logps/chosen": -32.44501495361328,
-      "logps/rejected": -34.933692932128906,
-      "loss": 0.2061,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3041638135910034,
-      "rewards/margins": 2.126009225845337,
-      "rewards/rejected": -0.8218452334403992,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.8027584552764893,
-      "logits/rejected": -1.805301308631897,
-      "logps/chosen": -31.277942657470703,
-      "logps/rejected": -36.393043518066406,
-      "loss": 0.1663,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2278459072113037,
-      "rewards/margins": 2.197005033493042,
-      "rewards/rejected": -0.9691593050956726,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.835371971130371,
-      "logits/rejected": -1.8288919925689697,
-      "logps/chosen": -34.11837387084961,
-      "logps/rejected": -37.60203552246094,
-      "loss": 0.2066,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.0293549299240112,
-      "rewards/margins": 2.101419687271118,
-      "rewards/rejected": -1.0720646381378174,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.813190221786499,
-      "logits/rejected": -1.807956337928772,
-      "logps/chosen": -29.670574188232422,
-      "logps/rejected": -37.203330993652344,
-      "loss": 0.2099,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.06589937210083,
-      "rewards/margins": 2.0366604328155518,
-      "rewards/rejected": -0.970760703086853,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.7904030084609985,
-      "logits/rejected": -1.779764175415039,
-      "logps/chosen": -30.418493270874023,
-      "logps/rejected": -30.5950984954834,
-      "loss": 0.209,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0641578435897827,
-      "rewards/margins": 1.8604481220245361,
-      "rewards/rejected": -0.796290397644043,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.1356685161590576,
-      "eval_logits/rejected": -2.130932331085205,
-      "eval_logps/chosen": -34.380455017089844,
-      "eval_logps/rejected": -38.083457946777344,
-      "eval_loss": 0.6776629090309143,
-      "eval_rewards/accuracies": 0.5917773842811584,
-      "eval_rewards/chosen": -0.2767248749732971,
-      "eval_rewards/margins": 0.17674396932125092,
-      "eval_rewards/rejected": -0.45346885919570923,
-      "eval_runtime": 145.9185,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.896877646446228,
-      "logits/rejected": -1.896228551864624,
-      "logps/chosen": -30.421009063720703,
-      "logps/rejected": -31.305410385131836,
-      "loss": 0.22,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.1225507259368896,
-      "rewards/margins": 1.9600980281829834,
-      "rewards/rejected": -0.8375473022460938,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.9368255138397217,
-      "logits/rejected": -1.9375327825546265,
-      "logps/chosen": -30.484455108642578,
-      "logps/rejected": -32.34059524536133,
-      "loss": 0.177,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.167297601699829,
-      "rewards/margins": 2.1071255207061768,
-      "rewards/rejected": -0.9398279190063477,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -1.9036754369735718,
-      "logits/rejected": -1.9070818424224854,
-      "logps/chosen": -30.951061248779297,
-      "logps/rejected": -28.893712997436523,
-      "loss": 0.227,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0632069110870361,
-      "rewards/margins": 1.7912060022354126,
-      "rewards/rejected": -0.7279989719390869,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.91388738155365,
-      "logits/rejected": -1.906005859375,
-      "logps/chosen": -29.750720977783203,
-      "logps/rejected": -34.767372131347656,
-      "loss": 0.1882,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1417419910430908,
-      "rewards/margins": 2.1554553508758545,
-      "rewards/rejected": -1.0137132406234741,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.8500654697418213,
-      "logits/rejected": -1.8609867095947266,
-      "logps/chosen": -32.1203727722168,
-      "logps/rejected": -33.35521697998047,
-      "loss": 0.1865,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1461875438690186,
-      "rewards/margins": 2.034356117248535,
-      "rewards/rejected": -0.888168454170227,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.9280004501342773,
-      "logits/rejected": -1.9186464548110962,
-      "logps/chosen": -32.13129425048828,
-      "logps/rejected": -37.68938446044922,
-      "loss": 0.2127,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.0955873727798462,
-      "rewards/margins": 2.181983470916748,
-      "rewards/rejected": -1.0863959789276123,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.835701584815979,
-      "logits/rejected": -1.837551474571228,
-      "logps/chosen": -34.348480224609375,
-      "logps/rejected": -33.834922790527344,
-      "loss": 0.1815,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3126420974731445,
-      "rewards/margins": 2.2222840785980225,
-      "rewards/rejected": -0.9096420407295227,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.9658441543579102,
-      "logits/rejected": -1.9667274951934814,
-      "logps/chosen": -31.538421630859375,
-      "logps/rejected": -35.54855728149414,
-      "loss": 0.1617,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2153749465942383,
-      "rewards/margins": 2.3616721630096436,
-      "rewards/rejected": -1.1462972164154053,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -1.9372714757919312,
-      "logits/rejected": -1.9413518905639648,
-      "logps/chosen": -29.44711685180664,
-      "logps/rejected": -34.69606399536133,
-      "loss": 0.2085,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.156960129737854,
-      "rewards/margins": 2.0755460262298584,
-      "rewards/rejected": -0.9185860753059387,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.9265114068984985,
-      "logits/rejected": -1.9317893981933594,
-      "logps/chosen": -32.267982482910156,
-      "logps/rejected": -31.293163299560547,
-      "loss": 0.2513,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0408172607421875,
-      "rewards/margins": 1.7540076971054077,
-      "rewards/rejected": -0.7131900191307068,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.1317365169525146,
-      "eval_logits/rejected": -2.126985549926758,
-      "eval_logps/chosen": -34.40774917602539,
-      "eval_logps/rejected": -38.09048080444336,
-      "eval_loss": 0.6897340416908264,
-      "eval_rewards/accuracies": 0.5830564498901367,
-      "eval_rewards/chosen": -0.2985561192035675,
-      "eval_rewards/margins": 0.1605302393436432,
-      "eval_rewards/rejected": -0.4590863585472107,
-      "eval_runtime": 145.9092,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -1.918264389038086,
-      "logits/rejected": -1.9183082580566406,
-      "logps/chosen": -31.37972068786621,
-      "logps/rejected": -31.99541664123535,
-      "loss": 0.1933,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.248342514038086,
-      "rewards/margins": 2.031942844390869,
-      "rewards/rejected": -0.783600389957428,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.7625865936279297,
-      "logits/rejected": -1.7643556594848633,
-      "logps/chosen": -33.15352249145508,
-      "logps/rejected": -34.140689849853516,
-      "loss": 0.2375,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.1291639804840088,
-      "rewards/margins": 2.008704900741577,
-      "rewards/rejected": -0.8795410394668579,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.7994060516357422,
-      "logits/rejected": -1.7968616485595703,
-      "logps/chosen": -29.835479736328125,
-      "logps/rejected": -32.159576416015625,
-      "loss": 0.2885,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.940991222858429,
-      "rewards/margins": 1.736061692237854,
-      "rewards/rejected": -0.7950705289840698,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -1.9358510971069336,
-      "logits/rejected": -1.9303522109985352,
-      "logps/chosen": -31.38555335998535,
-      "logps/rejected": -34.0118408203125,
-      "loss": 0.1779,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3233940601348877,
-      "rewards/margins": 2.1696605682373047,
-      "rewards/rejected": -0.8462664484977722,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.7935917377471924,
-      "logits/rejected": -1.783705472946167,
-      "logps/chosen": -32.11836624145508,
-      "logps/rejected": -32.378910064697266,
-      "loss": 0.2083,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.066697120666504,
-      "rewards/margins": 1.9359056949615479,
-      "rewards/rejected": -0.8692083358764648,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.8640037775039673,
-      "logits/rejected": -1.8639949560165405,
-      "logps/chosen": -32.08525848388672,
-      "logps/rejected": -32.31430435180664,
-      "loss": 0.1695,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.2771949768066406,
-      "rewards/margins": 2.167479991912842,
-      "rewards/rejected": -0.8902848958969116,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -1.9403139352798462,
-      "logits/rejected": -1.9347970485687256,
-      "logps/chosen": -32.035804748535156,
-      "logps/rejected": -34.015647888183594,
-      "loss": 0.186,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1921296119689941,
-      "rewards/margins": 2.0516738891601562,
-      "rewards/rejected": -0.8595443964004517,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.7985414266586304,
-      "logits/rejected": -1.806396484375,
-      "logps/chosen": -31.918773651123047,
-      "logps/rejected": -33.53784942626953,
-      "loss": 0.175,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1313796043395996,
-      "rewards/margins": 2.0725083351135254,
-      "rewards/rejected": -0.9411287307739258,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.86725652217865,
-      "logits/rejected": -1.8614327907562256,
-      "logps/chosen": -30.524410247802734,
-      "logps/rejected": -34.46692657470703,
-      "loss": 0.1668,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.319364309310913,
-      "rewards/margins": 2.266146659851074,
-      "rewards/rejected": -0.9467819929122925,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.8097419738769531,
-      "logits/rejected": -1.8118005990982056,
-      "logps/chosen": -32.011409759521484,
-      "logps/rejected": -34.247711181640625,
-      "loss": 0.1713,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.1832258701324463,
-      "rewards/margins": 2.373357057571411,
-      "rewards/rejected": -1.1901309490203857,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.131728410720825,
-      "eval_logits/rejected": -2.1269805431365967,
-      "eval_logps/chosen": -34.38140869140625,
-      "eval_logps/rejected": -38.09336471557617,
-      "eval_loss": 0.6780257821083069,
-      "eval_rewards/accuracies": 0.594684362411499,
-      "eval_rewards/chosen": -0.27748534083366394,
-      "eval_rewards/margins": 0.18390974402427673,
-      "eval_rewards/rejected": -0.4613950550556183,
-      "eval_runtime": 145.9474,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.7440783977508545,
-      "logits/rejected": -1.7430063486099243,
-      "logps/chosen": -34.408180236816406,
-      "logps/rejected": -34.03881072998047,
-      "loss": 0.1517,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.3154469728469849,
-      "rewards/margins": 2.3790054321289062,
-      "rewards/rejected": -1.0635584592819214,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.9152580499649048,
-      "logits/rejected": -1.9091880321502686,
-      "logps/chosen": -30.193584442138672,
-      "logps/rejected": -34.31517791748047,
-      "loss": 0.1516,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.2080823183059692,
-      "rewards/margins": 2.357390880584717,
-      "rewards/rejected": -1.149308443069458,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.7672933340072632,
-      "logits/rejected": -1.7558486461639404,
-      "logps/chosen": -32.647369384765625,
-      "logps/rejected": -34.192893981933594,
-      "loss": 0.1462,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3906490802764893,
-      "rewards/margins": 2.429901599884033,
-      "rewards/rejected": -1.0392528772354126,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.856805443763733,
-      "logits/rejected": -1.850097894668579,
-      "logps/chosen": -29.289770126342773,
-      "logps/rejected": -30.042938232421875,
-      "loss": 0.1928,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1283700466156006,
-      "rewards/margins": 2.0686745643615723,
-      "rewards/rejected": -0.9403042793273926,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.8852243423461914,
-      "logits/rejected": -1.8920814990997314,
-      "logps/chosen": -33.63641357421875,
-      "logps/rejected": -32.55451202392578,
-      "loss": 0.1642,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.4071805477142334,
-      "rewards/margins": 2.2361385822296143,
-      "rewards/rejected": -0.8289580345153809,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.8602968454360962,
-      "logits/rejected": -1.8607813119888306,
-      "logps/chosen": -31.587940216064453,
-      "logps/rejected": -34.207176208496094,
-      "loss": 0.1465,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1524927616119385,
-      "rewards/margins": 2.2663419246673584,
-      "rewards/rejected": -1.1138492822647095,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.800450086593628,
-      "logits/rejected": -1.7998168468475342,
-      "logps/chosen": -31.380176544189453,
-      "logps/rejected": -34.821746826171875,
-      "loss": 0.151,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3685301542282104,
-      "rewards/margins": 2.4094412326812744,
-      "rewards/rejected": -1.0409109592437744,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.686762809753418,
-      "logits/rejected": -1.6821815967559814,
-      "logps/chosen": -33.52080535888672,
-      "logps/rejected": -32.1031494140625,
-      "loss": 0.1522,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3733112812042236,
-      "rewards/margins": 2.314640760421753,
-      "rewards/rejected": -0.9413297772407532,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.8902908563613892,
-      "logits/rejected": -1.8748180866241455,
-      "logps/chosen": -32.840858459472656,
-      "logps/rejected": -36.358482360839844,
-      "loss": 0.1254,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2053766250610352,
-      "rewards/margins": 2.4799063205718994,
-      "rewards/rejected": -1.274530053138733,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.83368718624115,
-      "logits/rejected": -1.8386014699935913,
-      "logps/chosen": -31.894006729125977,
-      "logps/rejected": -37.62579345703125,
-      "loss": 0.1199,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.298893690109253,
-      "rewards/margins": 2.5169498920440674,
-      "rewards/rejected": -1.2180559635162354,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.131683349609375,
-      "eval_logits/rejected": -2.1269302368164062,
-      "eval_logps/chosen": -34.37530517578125,
-      "eval_logps/rejected": -38.09721374511719,
-      "eval_loss": 0.6739928126335144,
-      "eval_rewards/accuracies": 0.5980066657066345,
-      "eval_rewards/chosen": -0.2726014256477356,
-      "eval_rewards/margins": 0.19187170267105103,
-      "eval_rewards/rejected": -0.464473158121109,
-      "eval_runtime": 145.964,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.8072227239608765,
-      "logits/rejected": -1.8072770833969116,
-      "logps/chosen": -34.529273986816406,
-      "logps/rejected": -38.355064392089844,
-      "loss": 0.165,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.37179696559906,
-      "rewards/margins": 2.388556957244873,
-      "rewards/rejected": -1.0167598724365234,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -1.8941234350204468,
-      "logits/rejected": -1.8927080631256104,
-      "logps/chosen": -30.5273494720459,
-      "logps/rejected": -32.267601013183594,
-      "loss": 0.1563,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.2532024383544922,
-      "rewards/margins": 2.304598093032837,
-      "rewards/rejected": -1.0513951778411865,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.8009631633758545,
-      "logits/rejected": -1.8024797439575195,
-      "logps/chosen": -32.74066925048828,
-      "logps/rejected": -34.96391296386719,
-      "loss": 0.1456,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.306864619255066,
-      "rewards/margins": 2.3590540885925293,
-      "rewards/rejected": -1.052189588546753,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.8623323440551758,
-      "logits/rejected": -1.8611522912979126,
-      "logps/chosen": -29.76908302307129,
-      "logps/rejected": -33.332550048828125,
-      "loss": 0.1572,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1436678171157837,
-      "rewards/margins": 2.21612811088562,
-      "rewards/rejected": -1.072460412979126,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.877587914466858,
-      "logits/rejected": -1.862841248512268,
-      "logps/chosen": -32.96019744873047,
-      "logps/rejected": -37.72450637817383,
-      "loss": 0.153,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0851805210113525,
-      "rewards/margins": 2.253241539001465,
-      "rewards/rejected": -1.1680610179901123,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.8752319812774658,
-      "logits/rejected": -1.8875735998153687,
-      "logps/chosen": -30.424901962280273,
-      "logps/rejected": -33.42066192626953,
-      "loss": 0.1772,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.3148540258407593,
-      "rewards/margins": 2.2595856189727783,
-      "rewards/rejected": -0.9447315335273743,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -1.9719635248184204,
-      "logits/rejected": -1.971278429031372,
-      "logps/chosen": -30.404891967773438,
-      "logps/rejected": -37.444129943847656,
-      "loss": 0.1268,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.300431489944458,
-      "rewards/margins": 2.5910286903381348,
-      "rewards/rejected": -1.2905975580215454,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.8381322622299194,
-      "logits/rejected": -1.845894455909729,
-      "logps/chosen": -32.695735931396484,
-      "logps/rejected": -38.82695388793945,
-      "loss": 0.1845,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.1877927780151367,
-      "rewards/margins": 2.2896358966827393,
-      "rewards/rejected": -1.101843237876892,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.7921359539031982,
-      "logits/rejected": -1.7879730463027954,
-      "logps/chosen": -32.99811553955078,
-      "logps/rejected": -32.61560821533203,
-      "loss": 0.1972,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.4152759313583374,
-      "rewards/margins": 2.229846715927124,
-      "rewards/rejected": -0.8145707845687866,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.9499454498291016,
-      "logits/rejected": -1.9474738836288452,
-      "logps/chosen": -29.77544593811035,
-      "logps/rejected": -33.36935806274414,
-      "loss": 0.1578,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2168843746185303,
-      "rewards/margins": 2.1967949867248535,
-      "rewards/rejected": -0.979910671710968,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.131394863128662,
-      "eval_logits/rejected": -2.126645803451538,
-      "eval_logps/chosen": -34.39291763305664,
-      "eval_logps/rejected": -38.091670989990234,
-      "eval_loss": 0.6838791370391846,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": -0.28668972849845886,
-      "eval_rewards/margins": 0.17335131764411926,
-      "eval_rewards/rejected": -0.4600410461425781,
-      "eval_runtime": 145.9578,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -1.8938592672348022,
-      "logits/rejected": -1.8896372318267822,
-      "logps/chosen": -32.033363342285156,
-      "logps/rejected": -30.647994995117188,
-      "loss": 0.1724,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.2873361110687256,
-      "rewards/margins": 2.162166118621826,
-      "rewards/rejected": -0.8748300671577454,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.8500086069107056,
-      "logits/rejected": -1.8511066436767578,
-      "logps/chosen": -31.751754760742188,
-      "logps/rejected": -35.97992706298828,
-      "loss": 0.1799,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.29049813747406,
-      "rewards/margins": 2.1352338790893555,
-      "rewards/rejected": -0.8447355031967163,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.8036447763442993,
-      "logits/rejected": -1.7969812154769897,
-      "logps/chosen": -32.511940002441406,
-      "logps/rejected": -38.00764846801758,
-      "loss": 0.1569,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.303862452507019,
-      "rewards/margins": 2.358455181121826,
-      "rewards/rejected": -1.0545929670333862,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.8134548664093018,
-      "logits/rejected": -1.8079309463500977,
-      "logps/chosen": -27.75140380859375,
-      "logps/rejected": -32.93821334838867,
-      "loss": 0.1916,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.2494693994522095,
-      "rewards/margins": 2.2155184745788574,
-      "rewards/rejected": -0.9660491943359375,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.8413394689559937,
-      "logits/rejected": -1.8308031558990479,
-      "logps/chosen": -31.208871841430664,
-      "logps/rejected": -36.86098861694336,
-      "loss": 0.1533,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.0181248188018799,
-      "rewards/margins": 2.2786247730255127,
-      "rewards/rejected": -1.260500192642212,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.8131767511367798,
-      "logits/rejected": -1.8103145360946655,
-      "logps/chosen": -31.82034683227539,
-      "logps/rejected": -31.843425750732422,
-      "loss": 0.2074,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.1915438175201416,
-      "rewards/margins": 1.9536281824111938,
-      "rewards/rejected": -0.7620843052864075,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.8745816946029663,
-      "logits/rejected": -1.8810014724731445,
-      "logps/chosen": -30.35885238647461,
-      "logps/rejected": -34.228118896484375,
-      "loss": 0.1567,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.3682581186294556,
-      "rewards/margins": 2.380812883377075,
-      "rewards/rejected": -1.01255464553833,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -1.8742589950561523,
-      "logits/rejected": -1.8845386505126953,
-      "logps/chosen": -30.218097686767578,
-      "logps/rejected": -30.740955352783203,
-      "loss": 0.168,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.1817275285720825,
-      "rewards/margins": 2.0716397762298584,
-      "rewards/rejected": -0.889912486076355,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.8073488473892212,
-      "logits/rejected": -1.800286054611206,
-      "logps/chosen": -32.302825927734375,
-      "logps/rejected": -33.47874069213867,
-      "loss": 0.1842,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.082926869392395,
-      "rewards/margins": 2.097539186477661,
-      "rewards/rejected": -1.0146124362945557,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.8606147766113281,
-      "logits/rejected": -1.8575241565704346,
-      "logps/chosen": -32.205726623535156,
-      "logps/rejected": -34.68817138671875,
-      "loss": 0.1614,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1358753442764282,
-      "rewards/margins": 2.2782340049743652,
-      "rewards/rejected": -1.142358660697937,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.1312177181243896,
-      "eval_logits/rejected": -2.1264684200286865,
-      "eval_logps/chosen": -34.386234283447266,
-      "eval_logps/rejected": -38.08888244628906,
-      "eval_loss": 0.6819679737091064,
-      "eval_rewards/accuracies": 0.574335515499115,
-      "eval_rewards/chosen": -0.2813448905944824,
-      "eval_rewards/margins": 0.17646357417106628,
-      "eval_rewards/rejected": -0.4578084647655487,
-      "eval_runtime": 145.9567,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.9368705749511719,
-      "logits/rejected": -1.9374831914901733,
-      "logps/chosen": -28.38448715209961,
-      "logps/rejected": -34.06378173828125,
-      "loss": 0.2003,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.145431399345398,
-      "rewards/margins": 2.204097032546997,
-      "rewards/rejected": -1.0586655139923096,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.7932342290878296,
-      "logits/rejected": -1.7930355072021484,
-      "logps/chosen": -31.60140037536621,
-      "logps/rejected": -35.09097671508789,
-      "loss": 0.2089,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.1523739099502563,
-      "rewards/margins": 2.1261672973632812,
-      "rewards/rejected": -0.9737933874130249,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -1.943830132484436,
-      "logits/rejected": -1.9411506652832031,
-      "logps/chosen": -30.156597137451172,
-      "logps/rejected": -32.75658416748047,
-      "loss": 0.1839,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.053972840309143,
-      "rewards/margins": 2.0723206996917725,
-      "rewards/rejected": -1.018348217010498,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.9267715215682983,
-      "logits/rejected": -1.9278953075408936,
-      "logps/chosen": -29.27459716796875,
-      "logps/rejected": -30.95477294921875,
-      "loss": 0.1802,
-      "rewards/accuracies": 0.98333340883255,
-      "rewards/chosen": 1.0762757062911987,
-      "rewards/margins": 2.007948398590088,
-      "rewards/rejected": -0.9316726922988892,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.20605388523696305,
-      "train_runtime": 10804.1609,
-      "train_samples_per_second": 1.14,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.4538224170734356,
+      "train_runtime": 3252.427,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,