diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6944,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2413 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8664441108703613,
-      "logits/rejected": -1.8707667589187622,
-      "logps/chosen": -36.99871826171875,
-      "logps/rejected": -33.66032028198242,
-      "loss": 0.6698,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.0047301375307142735,
-      "rewards/margins": 0.022878725081682205,
-      "rewards/rejected": -0.018148591741919518,
+      "logits/chosen": -1.866829752922058,
+      "logits/rejected": -1.8711390495300293,
+      "logps/chosen": -36.97040557861328,
+      "logps/rejected": -33.66280746459961,
+      "loss": 0.9586,
+      "rewards/accuracies": 0.5972222089767456,
+      "rewards/chosen": 0.021715592592954636,
+      "rewards/margins": 0.041353289037942886,
+      "rewards/rejected": -0.01963769644498825,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9972795248031616,
-      "logits/rejected": -1.999922513961792,
-      "logps/chosen": -29.64887046813965,
-      "logps/rejected": -29.054330825805664,
-      "loss": 0.756,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.0040070666000247,
-      "rewards/margins": -0.009474089369177818,
-      "rewards/rejected": 0.0054670218378305435,
+      "logits/chosen": -1.9976301193237305,
+      "logits/rejected": -2.0002708435058594,
+      "logps/chosen": -29.64252281188965,
+      "logps/rejected": -29.05857276916504,
+      "loss": 1.0031,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00019791116937994957,
+      "rewards/margins": -0.003119796048849821,
+      "rewards/rejected": 0.0029218837153166533,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9204769134521484,
-      "logits/rejected": -1.9177768230438232,
-      "logps/chosen": -31.404077529907227,
-      "logps/rejected": -33.22195053100586,
-      "loss": 0.7112,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.007212291471660137,
-      "rewards/margins": 0.009304212406277657,
-      "rewards/rejected": -0.0020919169764965773,
+      "logits/chosen": -1.9210872650146484,
+      "logits/rejected": -1.918402910232544,
+      "logps/chosen": -31.397838592529297,
+      "logps/rejected": -33.22719192504883,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.010952453128993511,
+      "rewards/margins": 0.01619068905711174,
+      "rewards/rejected": -0.005238235928118229,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0172927379608154,
-      "logits/rejected": -2.008545398712158,
-      "logps/chosen": -32.57653045654297,
-      "logps/rejected": -32.520172119140625,
-      "loss": 0.7334,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00016416943981312215,
-      "rewards/margins": 0.004665405489504337,
-      "rewards/rejected": -0.004501235671341419,
+      "logits/chosen": -2.017606258392334,
+      "logits/rejected": -2.0088634490966797,
+      "logps/chosen": -32.566612243652344,
+      "logps/rejected": -32.52539825439453,
+      "loss": 0.9862,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.006120447069406509,
+      "rewards/margins": 0.013754432089626789,
+      "rewards/rejected": -0.007633985485881567,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8620569705963135,
-      "logits/rejected": -1.8512712717056274,
-      "logps/chosen": -33.55234146118164,
-      "logps/rejected": -35.454689025878906,
-      "loss": 0.7458,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.003082751529291272,
-      "rewards/margins": 0.004490534774959087,
-      "rewards/rejected": -0.0014077809173613787,
+      "logits/chosen": -1.8625805377960205,
+      "logits/rejected": -1.8518139123916626,
+      "logps/chosen": -33.56818771362305,
+      "logps/rejected": -35.4713020324707,
+      "loss": 0.9951,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0064262161031365395,
+      "rewards/margins": 0.004948456306010485,
+      "rewards/rejected": -0.011374671943485737,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.940212607383728,
-      "logits/rejected": -1.9421584606170654,
-      "logps/chosen": -32.56269454956055,
-      "logps/rejected": -33.21684265136719,
-      "loss": 0.6785,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.022492902353405952,
-      "rewards/margins": 0.04977109283208847,
-      "rewards/rejected": -0.027278194203972816,
+      "logits/chosen": -1.9413652420043945,
+      "logits/rejected": -1.9432977437973022,
+      "logps/chosen": -32.551727294921875,
+      "logps/rejected": -33.218013763427734,
+      "loss": 0.9525,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.02907358668744564,
+      "rewards/margins": 0.05705299973487854,
+      "rewards/rejected": -0.02797941491007805,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0728747844696045,
-      "logits/rejected": -2.0778398513793945,
-      "logps/chosen": -33.98863983154297,
-      "logps/rejected": -36.62779235839844,
-      "loss": 0.7641,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -9.316801879322156e-05,
-      "rewards/margins": 0.035260457545518875,
-      "rewards/rejected": -0.035353630781173706,
+      "logits/chosen": -2.0722384452819824,
+      "logits/rejected": -2.0772085189819336,
+      "logps/chosen": -33.995296478271484,
+      "logps/rejected": -36.63286590576172,
+      "loss": 0.9657,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.004085049033164978,
+      "rewards/margins": 0.034312374889850616,
+      "rewards/rejected": -0.038397423923015594,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.935132384300232,
-      "logits/rejected": -1.9382661581039429,
-      "logps/chosen": -34.321903228759766,
-      "logps/rejected": -34.638282775878906,
-      "loss": 0.6095,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.06354494392871857,
-      "rewards/margins": 0.09106875956058502,
-      "rewards/rejected": -0.027523810043931007,
+      "logits/chosen": -1.9325292110443115,
+      "logits/rejected": -1.93563973903656,
+      "logps/chosen": -34.35693359375,
+      "logps/rejected": -34.67926788330078,
+      "loss": 0.9069,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.042526502162218094,
+      "rewards/margins": 0.09463920444250107,
+      "rewards/rejected": -0.05211270600557327,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9444854259490967,
-      "logits/rejected": -1.9490203857421875,
-      "logps/chosen": -32.41173553466797,
-      "logps/rejected": -32.337852478027344,
-      "loss": 0.7499,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.033411428332328796,
-      "rewards/margins": 0.020780622959136963,
-      "rewards/rejected": 0.012630810961127281,
+      "logits/chosen": -1.9406629800796509,
+      "logits/rejected": -1.9451707601547241,
+      "logps/chosen": -32.38554000854492,
+      "logps/rejected": -32.36783218383789,
+      "loss": 0.9455,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.04912734776735306,
+      "rewards/margins": 0.05448007583618164,
+      "rewards/rejected": -0.005352728068828583,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.042165994644165,
-      "logits/rejected": -2.0401768684387207,
-      "logps/chosen": -32.13690185546875,
-      "logps/rejected": -31.2872314453125,
-      "loss": 0.5903,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.06236031651496887,
-      "rewards/margins": 0.0749734491109848,
-      "rewards/rejected": -0.012613129802048206,
+      "logits/chosen": -2.037841320037842,
+      "logits/rejected": -2.0358526706695557,
+      "logps/chosen": -32.14287185668945,
+      "logps/rejected": -31.300832748413086,
+      "loss": 0.9204,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.058778904378414154,
+      "rewards/margins": 0.07955379039049149,
+      "rewards/rejected": -0.02077488601207733,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2366247177124023,
-      "eval_logits/rejected": -2.231769323348999,
-      "eval_logps/chosen": -34.01411819458008,
-      "eval_logps/rejected": -37.50873947143555,
-      "eval_loss": 0.7449910640716553,
-      "eval_rewards/accuracies": 0.5240863561630249,
-      "eval_rewards/chosen": 0.01226096786558628,
-      "eval_rewards/margins": 0.007533104624599218,
-      "eval_rewards/rejected": 0.0047278618440032005,
-      "eval_runtime": 145.783,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.2316644191741943,
+      "eval_logits/rejected": -2.2268166542053223,
+      "eval_logps/chosen": -34.03582000732422,
+      "eval_logps/rejected": -37.548709869384766,
+      "eval_loss": 0.9816102385520935,
+      "eval_rewards/accuracies": 0.5394518375396729,
+      "eval_rewards/chosen": -0.0007606110884808004,
+      "eval_rewards/margins": 0.018493397161364555,
+      "eval_rewards/rejected": -0.019254004582762718,
+      "eval_runtime": 145.9929,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9979403018951416,
-      "logits/rejected": -1.9955459833145142,
-      "logps/chosen": -33.12257766723633,
-      "logps/rejected": -33.99717712402344,
-      "loss": 0.8547,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.07274619489908218,
-      "rewards/margins": 0.05342693254351616,
-      "rewards/rejected": 0.019319258630275726,
+      "logits/chosen": -1.9917171001434326,
+      "logits/rejected": -1.9893367290496826,
+      "logps/chosen": -33.120052337646484,
+      "logps/rejected": -34.02777099609375,
+      "loss": 0.9329,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0742575079202652,
+      "rewards/margins": 0.0732978954911232,
+      "rewards/rejected": 0.0009596304735168815,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.009084463119507,
-      "logits/rejected": -2.0007264614105225,
-      "logps/chosen": -32.33968734741211,
-      "logps/rejected": -32.13513946533203,
-      "loss": 0.7104,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06312941759824753,
-      "rewards/margins": 0.047309644520282745,
-      "rewards/rejected": 0.015819773077964783,
+      "logits/chosen": -2.0032849311828613,
+      "logits/rejected": -1.9949623346328735,
+      "logps/chosen": -32.30955123901367,
+      "logps/rejected": -32.144927978515625,
+      "loss": 0.9308,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0812125876545906,
+      "rewards/margins": 0.07126720249652863,
+      "rewards/rejected": 0.009945395402610302,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0365753173828125,
-      "logits/rejected": -2.028564453125,
-      "logps/chosen": -30.363454818725586,
-      "logps/rejected": -32.06770706176758,
-      "loss": 0.71,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.06735612452030182,
-      "rewards/margins": 0.07617705315351486,
-      "rewards/rejected": -0.008820934221148491,
+      "logits/chosen": -2.031322956085205,
+      "logits/rejected": -2.023355007171631,
+      "logps/chosen": -30.32772445678711,
+      "logps/rejected": -32.05691909790039,
+      "loss": 0.9282,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.08879294246435165,
+      "rewards/margins": 0.09113965928554535,
+      "rewards/rejected": -0.002346712863072753,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9657623767852783,
-      "logits/rejected": -1.976011872291565,
-      "logps/chosen": -31.213130950927734,
-      "logps/rejected": -32.592655181884766,
-      "loss": 0.6309,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.12391922622919083,
-      "rewards/margins": 0.1585826873779297,
-      "rewards/rejected": -0.03466347977519035,
+      "logits/chosen": -1.9619598388671875,
+      "logits/rejected": -1.9721866846084595,
+      "logps/chosen": -31.2375431060791,
+      "logps/rejected": -32.572418212890625,
+      "loss": 0.8702,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1092713475227356,
+      "rewards/margins": 0.13179358839988708,
+      "rewards/rejected": -0.02252225950360298,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8778575658798218,
-      "logits/rejected": -1.879020094871521,
-      "logps/chosen": -33.998287200927734,
-      "logps/rejected": -34.83546829223633,
-      "loss": 0.566,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.11664185672998428,
-      "rewards/margins": 0.1635705530643463,
-      "rewards/rejected": -0.04692870378494263,
+      "logits/chosen": -1.8724457025527954,
+      "logits/rejected": -1.8736213445663452,
+      "logps/chosen": -33.905731201171875,
+      "logps/rejected": -34.80728530883789,
+      "loss": 0.818,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.17217601835727692,
+      "rewards/margins": 0.20219452679157257,
+      "rewards/rejected": -0.03001854196190834,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9302040338516235,
-      "logits/rejected": -1.9267680644989014,
-      "logps/chosen": -36.0306510925293,
-      "logps/rejected": -32.73199462890625,
-      "loss": 0.5795,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.08461670577526093,
-      "rewards/margins": 0.09016285836696625,
-      "rewards/rejected": -0.005546141881495714,
+      "logits/chosen": -1.923604965209961,
+      "logits/rejected": -1.9202001094818115,
+      "logps/chosen": -36.00878143310547,
+      "logps/rejected": -32.71224594116211,
+      "loss": 0.9086,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.09773740917444229,
+      "rewards/margins": 0.09143415838479996,
+      "rewards/rejected": 0.0063032531179487705,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.031188726425171,
-      "logits/rejected": -2.0238261222839355,
-      "logps/chosen": -33.55583953857422,
-      "logps/rejected": -31.381237030029297,
-      "loss": 0.5041,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.14521491527557373,
-      "rewards/margins": 0.16284644603729248,
-      "rewards/rejected": -0.017631549388170242,
+      "logits/chosen": -2.024691343307495,
+      "logits/rejected": -2.0173799991607666,
+      "logps/chosen": -33.495445251464844,
+      "logps/rejected": -31.415653228759766,
+      "loss": 0.7882,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.18144862353801727,
+      "rewards/margins": 0.21973037719726562,
+      "rewards/rejected": -0.03828175365924835,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.037135362625122,
-      "logits/rejected": -2.0424118041992188,
-      "logps/chosen": -32.33055114746094,
-      "logps/rejected": -32.500144958496094,
-      "loss": 0.5257,
+      "logits/chosen": -2.031024694442749,
+      "logits/rejected": -2.036294937133789,
+      "logps/chosen": -32.22355270385742,
+      "logps/rejected": -32.47669219970703,
+      "loss": 0.8225,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.12854143977165222,
-      "rewards/margins": 0.13754449784755707,
-      "rewards/rejected": -0.009003056213259697,
+      "rewards/chosen": 0.19274193048477173,
+      "rewards/margins": 0.1876741349697113,
+      "rewards/rejected": 0.00506778946146369,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0384888648986816,
-      "logits/rejected": -2.0357367992401123,
-      "logps/chosen": -31.326950073242188,
-      "logps/rejected": -31.338672637939453,
-      "loss": 0.6504,
+      "logits/chosen": -2.0325100421905518,
+      "logits/rejected": -2.029745578765869,
+      "logps/chosen": -31.281234741210938,
+      "logps/rejected": -31.360708236694336,
+      "loss": 0.8564,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09720095247030258,
-      "rewards/margins": 0.10947537422180176,
-      "rewards/rejected": -0.012274429202079773,
+      "rewards/chosen": 0.12463297694921494,
+      "rewards/margins": 0.15012845396995544,
+      "rewards/rejected": -0.025495493784546852,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.908755898475647,
-      "logits/rejected": -1.9134235382080078,
-      "logps/chosen": -31.36275863647461,
-      "logps/rejected": -32.82594680786133,
-      "loss": 0.59,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14611336588859558,
-      "rewards/margins": 0.16712167859077454,
-      "rewards/rejected": -0.02100830152630806,
+      "logits/chosen": -1.902600646018982,
+      "logits/rejected": -1.9072542190551758,
+      "logps/chosen": -31.33133316040039,
+      "logps/rejected": -32.85138702392578,
+      "loss": 0.8045,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.16497036814689636,
+      "rewards/margins": 0.2012440711259842,
+      "rewards/rejected": -0.03627369925379753,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.233200788497925,
-      "eval_logits/rejected": -2.228351354598999,
-      "eval_logps/chosen": -34.069149017333984,
-      "eval_logps/rejected": -37.57521057128906,
-      "eval_loss": 0.750046968460083,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": -0.020758595317602158,
-      "eval_rewards/margins": 0.014396170154213905,
-      "eval_rewards/rejected": -0.035154763609170914,
-      "eval_runtime": 145.6956,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2286624908447266,
+      "eval_logits/rejected": -2.2238309383392334,
+      "eval_logps/chosen": -34.062259674072266,
+      "eval_logps/rejected": -37.57956314086914,
+      "eval_loss": 0.9791061282157898,
+      "eval_rewards/accuracies": 0.5361295938491821,
+      "eval_rewards/chosen": -0.016622914001345634,
+      "eval_rewards/margins": 0.021141981706023216,
+      "eval_rewards/rejected": -0.03776489570736885,
+      "eval_runtime": 145.8001,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0207877159118652,
-      "logits/rejected": -2.0314865112304688,
-      "logps/chosen": -31.7946834564209,
-      "logps/rejected": -33.94416046142578,
-      "loss": 0.544,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.10597337782382965,
-      "rewards/margins": 0.1548517644405365,
-      "rewards/rejected": -0.04887838289141655,
+      "logits/chosen": -2.0155138969421387,
+      "logits/rejected": -2.0261716842651367,
+      "logps/chosen": -31.7918758392334,
+      "logps/rejected": -33.9483528137207,
+      "loss": 0.8508,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10765652358531952,
+      "rewards/margins": 0.15904827415943146,
+      "rewards/rejected": -0.051391761749982834,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9139292240142822,
-      "logits/rejected": -1.9287306070327759,
-      "logps/chosen": -29.91892433166504,
-      "logps/rejected": -31.5737247467041,
-      "loss": 0.5694,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.11556863784790039,
-      "rewards/margins": 0.12489677965641022,
-      "rewards/rejected": -0.00932813435792923,
+      "logits/chosen": -1.908071756362915,
+      "logits/rejected": -1.9228509664535522,
+      "logps/chosen": -29.81874656677246,
+      "logps/rejected": -31.65224266052246,
+      "loss": 0.7758,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.175676628947258,
+      "rewards/margins": 0.23211655020713806,
+      "rewards/rejected": -0.05643991753458977,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9701712131500244,
-      "logits/rejected": -1.974172592163086,
-      "logps/chosen": -33.17791748046875,
-      "logps/rejected": -31.603836059570312,
-      "loss": 0.4933,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1437317281961441,
-      "rewards/margins": 0.1764504611492157,
-      "rewards/rejected": -0.03271873667836189,
+      "logits/chosen": -1.9642655849456787,
+      "logits/rejected": -1.9682365655899048,
+      "logps/chosen": -33.13427734375,
+      "logps/rejected": -31.658504486083984,
+      "loss": 0.7961,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.16991479694843292,
+      "rewards/margins": 0.23543640971183777,
+      "rewards/rejected": -0.06552163511514664,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9691722393035889,
-      "logits/rejected": -1.9472358226776123,
-      "logps/chosen": -33.92795181274414,
-      "logps/rejected": -35.06544876098633,
-      "loss": 0.4966,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.13547900319099426,
-      "rewards/margins": 0.20427203178405762,
-      "rewards/rejected": -0.06879300624132156,
+      "logits/chosen": -1.9623470306396484,
+      "logits/rejected": -1.9405139684677124,
+      "logps/chosen": -33.870643615722656,
+      "logps/rejected": -35.13296127319336,
+      "loss": 0.7551,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16986075043678284,
+      "rewards/margins": 0.279163658618927,
+      "rewards/rejected": -0.10930290073156357,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.010617971420288,
-      "logits/rejected": -2.0072851181030273,
-      "logps/chosen": -32.785011291503906,
-      "logps/rejected": -36.283348083496094,
-      "loss": 0.5823,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.08403309434652328,
-      "rewards/margins": 0.11908762156963348,
-      "rewards/rejected": -0.035054538398981094,
+      "logits/chosen": -2.003516674041748,
+      "logits/rejected": -2.0002071857452393,
+      "logps/chosen": -32.743778228759766,
+      "logps/rejected": -36.293540954589844,
+      "loss": 0.867,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10877454280853271,
+      "rewards/margins": 0.14994443953037262,
+      "rewards/rejected": -0.041169874370098114,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8787453174591064,
-      "logits/rejected": -1.8763244152069092,
-      "logps/chosen": -34.000633239746094,
-      "logps/rejected": -35.53338623046875,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.11265683174133301,
-      "rewards/margins": 0.1299169510602951,
-      "rewards/rejected": -0.017260130494832993,
+      "logits/chosen": -1.8702592849731445,
+      "logits/rejected": -1.867851972579956,
+      "logps/chosen": -33.99380111694336,
+      "logps/rejected": -35.564971923828125,
+      "loss": 0.85,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.11675956100225449,
+      "rewards/margins": 0.15297001600265503,
+      "rewards/rejected": -0.03621045500040054,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.864057183265686,
-      "logits/rejected": -1.861581802368164,
-      "logps/chosen": -34.23812484741211,
-      "logps/rejected": -31.777530670166016,
-      "loss": 0.6307,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.09568403661251068,
-      "rewards/margins": 0.09940369427204132,
-      "rewards/rejected": -0.003719669533893466,
+      "logits/chosen": -1.8547359704971313,
+      "logits/rejected": -1.8523473739624023,
+      "logps/chosen": -34.247703552246094,
+      "logps/rejected": -31.8588809967041,
+      "loss": 0.8606,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0899367406964302,
+      "rewards/margins": 0.14246629178524017,
+      "rewards/rejected": -0.05252955108880997,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9680726528167725,
-      "logits/rejected": -1.957493782043457,
-      "logps/chosen": -35.05353546142578,
-      "logps/rejected": -31.8691463470459,
-      "loss": 0.4252,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.18186399340629578,
-      "rewards/margins": 0.19875726103782654,
-      "rewards/rejected": -0.01689324900507927,
+      "logits/chosen": -1.9580624103546143,
+      "logits/rejected": -1.9475984573364258,
+      "logps/chosen": -35.032501220703125,
+      "logps/rejected": -31.907541275024414,
+      "loss": 0.7731,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.19448420405387878,
+      "rewards/margins": 0.23441271483898163,
+      "rewards/rejected": -0.039928533136844635,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0636587142944336,
-      "logits/rejected": -2.048659563064575,
-      "logps/chosen": -30.76055908203125,
-      "logps/rejected": -32.65589141845703,
-      "loss": 0.6667,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09814020991325378,
-      "rewards/margins": 0.10057313740253448,
-      "rewards/rejected": -0.0024329207371920347,
+      "logits/chosen": -2.0538430213928223,
+      "logits/rejected": -2.038924217224121,
+      "logps/chosen": -30.76409912109375,
+      "logps/rejected": -32.64435958862305,
+      "loss": 0.9184,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09601466357707977,
+      "rewards/margins": 0.09152902662754059,
+      "rewards/rejected": 0.004485635552555323,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9355440139770508,
-      "logits/rejected": -1.9329853057861328,
-      "logps/chosen": -32.57345199584961,
-      "logps/rejected": -30.86557388305664,
-      "loss": 0.3871,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.21219508349895477,
-      "rewards/margins": 0.24145305156707764,
-      "rewards/rejected": -0.029257934540510178,
+      "logits/chosen": -1.9241282939910889,
+      "logits/rejected": -1.9215872287750244,
+      "logps/chosen": -32.469078063964844,
+      "logps/rejected": -30.92861557006836,
+      "loss": 0.7186,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2748206555843353,
+      "rewards/margins": 0.34190455079078674,
+      "rewards/rejected": -0.06708388030529022,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.233344554901123,
-      "eval_logits/rejected": -2.2284841537475586,
-      "eval_logps/chosen": -34.06221008300781,
-      "eval_logps/rejected": -37.56574249267578,
-      "eval_loss": 0.7655012607574463,
+      "eval_logits/chosen": -2.224278211593628,
+      "eval_logits/rejected": -2.219449996948242,
+      "eval_logps/chosen": -34.111026763916016,
+      "eval_logps/rejected": -37.63855743408203,
+      "eval_loss": 0.9731553196907043,
       "eval_rewards/accuracies": 0.5070598125457764,
-      "eval_rewards/chosen": -0.016592998057603836,
-      "eval_rewards/margins": 0.012880506925284863,
-      "eval_rewards/rejected": -0.029473504051566124,
-      "eval_runtime": 145.6769,
-      "eval_samples_per_second": 2.355,
+      "eval_rewards/chosen": -0.045885536819696426,
+      "eval_rewards/margins": 0.02727569453418255,
+      "eval_rewards/rejected": -0.07316123694181442,
+      "eval_runtime": 145.8155,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -1.9183489084243774,
-      "logits/rejected": -1.9150705337524414,
-      "logps/chosen": -31.377817153930664,
-      "logps/rejected": -33.78107833862305,
-      "loss": 0.573,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.13532012701034546,
-      "rewards/margins": 0.1586771160364151,
-      "rewards/rejected": -0.02335697039961815,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.9088821411132812,
+      "logits/rejected": -1.9056317806243896,
+      "logps/chosen": -31.343881607055664,
+      "logps/rejected": -33.82904052734375,
+      "loss": 0.8131,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.15568041801452637,
+      "rewards/margins": 0.20781342685222626,
+      "rewards/rejected": -0.05213301628828049,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9682114124298096,
-      "logits/rejected": -1.9559390544891357,
-      "logps/chosen": -34.42866134643555,
-      "logps/rejected": -33.64933395385742,
-      "loss": 0.47,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.10044805705547333,
-      "rewards/margins": 0.16674570739269257,
-      "rewards/rejected": -0.06629763543605804,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.9583854675292969,
+      "logits/rejected": -1.9461708068847656,
+      "logps/chosen": -34.326744079589844,
+      "logps/rejected": -33.68773651123047,
+      "loss": 0.763,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1615992784500122,
+      "rewards/margins": 0.250935435295105,
+      "rewards/rejected": -0.08933614194393158,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -2.004417896270752,
-      "logits/rejected": -2.0029640197753906,
-      "logps/chosen": -33.30437469482422,
-      "logps/rejected": -32.512184143066406,
-      "loss": 0.5649,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.10884843021631241,
-      "rewards/margins": 0.12841013073921204,
-      "rewards/rejected": -0.019561681896448135,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -1.9934914112091064,
+      "logits/rejected": -1.9920612573623657,
+      "logps/chosen": -33.20575714111328,
+      "logps/rejected": -32.5851936340332,
+      "loss": 0.7771,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.16801992058753967,
+      "rewards/margins": 0.23138853907585144,
+      "rewards/rejected": -0.06336863338947296,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -2.091501474380493,
-      "logits/rejected": -2.075732707977295,
-      "logps/chosen": -33.85096740722656,
-      "logps/rejected": -33.104610443115234,
-      "loss": 0.5076,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.19705387949943542,
-      "rewards/margins": 0.18899771571159363,
-      "rewards/rejected": 0.008056174032390118,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.080026149749756,
+      "logits/rejected": -2.064359188079834,
+      "logps/chosen": -33.830657958984375,
+      "logps/rejected": -33.15457534790039,
+      "loss": 0.7811,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.20923766493797302,
+      "rewards/margins": 0.23116116225719452,
+      "rewards/rejected": -0.021923482418060303,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.964172124862671,
-      "logits/rejected": -1.96323561668396,
-      "logps/chosen": -32.96615219116211,
-      "logps/rejected": -32.529842376708984,
-      "loss": 0.4871,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.18956632912158966,
-      "rewards/margins": 0.2283860146999359,
-      "rewards/rejected": -0.03881971538066864,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.952505111694336,
+      "logits/rejected": -1.95168936252594,
+      "logps/chosen": -32.850311279296875,
+      "logps/rejected": -32.56792449951172,
+      "loss": 0.7213,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.25907084345817566,
+      "rewards/margins": 0.3207371234893799,
+      "rewards/rejected": -0.061666231602430344,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -1.919790267944336,
-      "logits/rejected": -1.9300991296768188,
-      "logps/chosen": -31.94921875,
-      "logps/rejected": -35.33544921875,
-      "loss": 0.5992,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.15328440070152283,
-      "rewards/margins": 0.18129965662956238,
-      "rewards/rejected": -0.028015267103910446,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9081252813339233,
+      "logits/rejected": -1.918378472328186,
+      "logps/chosen": -31.902362823486328,
+      "logps/rejected": -35.33353805541992,
+      "loss": 0.7965,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1813959777355194,
+      "rewards/margins": 0.20826482772827148,
+      "rewards/rejected": -0.026868879795074463,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -2.0574381351470947,
-      "logits/rejected": -2.050945520401001,
-      "logps/chosen": -33.3778076171875,
-      "logps/rejected": -29.26238441467285,
-      "loss": 0.4666,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.16449405252933502,
-      "rewards/margins": 0.186069518327713,
-      "rewards/rejected": -0.021575476974248886,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.0479800701141357,
+      "logits/rejected": -2.0415284633636475,
+      "logps/chosen": -33.33736038208008,
+      "logps/rejected": -29.26900863647461,
+      "loss": 0.8,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.18876628577709198,
+      "rewards/margins": 0.21431729197502136,
+      "rewards/rejected": -0.025551024824380875,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -1.9166724681854248,
-      "logits/rejected": -1.9188873767852783,
-      "logps/chosen": -33.89116668701172,
-      "logps/rejected": -30.885635375976562,
-      "loss": 0.4244,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2087288796901703,
-      "rewards/margins": 0.21653230488300323,
-      "rewards/rejected": -0.007803438697010279,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9078292846679688,
+      "logits/rejected": -1.9100587368011475,
+      "logps/chosen": -33.90921401977539,
+      "logps/rejected": -30.994491577148438,
+      "loss": 0.7663,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.19790422916412354,
+      "rewards/margins": 0.271023154258728,
+      "rewards/rejected": -0.07311896234750748,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.9459316730499268,
-      "logits/rejected": -1.9447450637817383,
-      "logps/chosen": -34.006561279296875,
-      "logps/rejected": -35.8227424621582,
-      "loss": 0.3945,
-      "rewards/accuracies": 0.7958333492279053,
-      "rewards/chosen": 0.22693590819835663,
-      "rewards/margins": 0.3269794285297394,
-      "rewards/rejected": -0.10004355013370514,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.8829723596572876,
-      "logits/rejected": -1.8746156692504883,
-      "logps/chosen": -31.377273559570312,
-      "logps/rejected": -36.17646408081055,
-      "loss": 0.3315,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.23802152276039124,
-      "rewards/margins": 0.3795410692691803,
-      "rewards/rejected": -0.14151951670646667,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.231396436691284,
-      "eval_logits/rejected": -2.226522207260132,
-      "eval_logps/chosen": -34.036094665527344,
-      "eval_logps/rejected": -37.56599044799805,
-      "eval_loss": 0.7478801608085632,
-      "eval_rewards/accuracies": 0.5514950156211853,
-      "eval_rewards/chosen": -0.0009239073260687292,
-      "eval_rewards/margins": 0.02869875729084015,
-      "eval_rewards/rejected": -0.02962266281247139,
-      "eval_runtime": 146.4126,
-      "eval_samples_per_second": 2.343,
-      "eval_steps_per_second": 0.294,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -2.0595316886901855,
-      "logits/rejected": -2.062485933303833,
-      "logps/chosen": -32.621891021728516,
-      "logps/rejected": -35.11821365356445,
-      "loss": 0.5251,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.2592288553714752,
-      "rewards/margins": 0.45444852113723755,
-      "rewards/rejected": -0.19521966576576233,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.903078317642212,
-      "logits/rejected": -1.9016958475112915,
-      "logps/chosen": -28.610631942749023,
-      "logps/rejected": -32.66521453857422,
-      "loss": 0.3322,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.27415698766708374,
-      "rewards/margins": 0.48268675804138184,
-      "rewards/rejected": -0.2085297405719757,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.8610217571258545,
-      "logits/rejected": -1.854017972946167,
-      "logps/chosen": -33.55864334106445,
-      "logps/rejected": -34.378604888916016,
-      "loss": 0.3646,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3092653155326843,
-      "rewards/margins": 0.389264315366745,
-      "rewards/rejected": -0.07999899238348007,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -2.0365495681762695,
-      "logits/rejected": -2.0314345359802246,
-      "logps/chosen": -31.39179039001465,
-      "logps/rejected": -32.26957702636719,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.23490269482135773,
-      "rewards/margins": 0.41608113050460815,
-      "rewards/rejected": -0.18117846548557281,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -1.92501962184906,
-      "logits/rejected": -1.9292068481445312,
-      "logps/chosen": -34.05653381347656,
-      "logps/rejected": -32.91465759277344,
-      "loss": 0.551,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.27558597922325134,
-      "rewards/margins": 0.4711950421333313,
-      "rewards/rejected": -0.19560904800891876,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -1.934442162513733,
-      "logits/rejected": -1.9286327362060547,
-      "logps/chosen": -34.91238784790039,
-      "logps/rejected": -35.231353759765625,
-      "loss": 0.3759,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3042834401130676,
-      "rewards/margins": 0.47363799810409546,
-      "rewards/rejected": -0.16935452818870544,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.9695565700531006,
-      "logits/rejected": -1.9690759181976318,
-      "logps/chosen": -33.69108581542969,
-      "logps/rejected": -34.42035675048828,
-      "loss": 0.2728,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.2863459885120392,
-      "rewards/margins": 0.42727118730545044,
-      "rewards/rejected": -0.14092518389225006,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -2.0204994678497314,
-      "logits/rejected": -2.0016019344329834,
-      "logps/chosen": -33.737281799316406,
-      "logps/rejected": -33.34347152709961,
-      "loss": 0.4172,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.2540892958641052,
-      "rewards/margins": 0.4181840419769287,
-      "rewards/rejected": -0.1640947312116623,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9881887435913086,
-      "logits/rejected": -1.9876505136489868,
-      "logps/chosen": -31.16998291015625,
-      "logps/rejected": -31.431163787841797,
-      "loss": 0.315,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.264793336391449,
-      "rewards/margins": 0.37586522102355957,
-      "rewards/rejected": -0.1110718846321106,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.9539533853530884,
-      "logits/rejected": -1.946997880935669,
-      "logps/chosen": -32.230567932128906,
-      "logps/rejected": -34.859554290771484,
-      "loss": 0.3857,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.2637366056442261,
-      "rewards/margins": 0.4034242630004883,
-      "rewards/rejected": -0.139687642455101,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.2342920303344727,
-      "eval_logits/rejected": -2.2294206619262695,
-      "eval_logps/chosen": -34.03615951538086,
-      "eval_logps/rejected": -37.557979583740234,
-      "eval_loss": 0.7601029276847839,
-      "eval_rewards/accuracies": 0.5577242374420166,
-      "eval_rewards/chosen": -0.0009655931498855352,
-      "eval_rewards/margins": 0.023849941790103912,
-      "eval_rewards/rejected": -0.02481553703546524,
-      "eval_runtime": 145.9216,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.9524669647216797,
-      "logits/rejected": -1.9562795162200928,
-      "logps/chosen": -28.682544708251953,
-      "logps/rejected": -33.07499313354492,
-      "loss": 0.3645,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.29536908864974976,
-      "rewards/margins": 0.42479196190834045,
-      "rewards/rejected": -0.12942291796207428,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -1.9641717672348022,
-      "logits/rejected": -1.9746614694595337,
-      "logps/chosen": -32.76649475097656,
-      "logps/rejected": -30.801523208618164,
-      "loss": 0.2418,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.2975025177001953,
-      "rewards/margins": 0.3874046206474304,
-      "rewards/rejected": -0.0899021252989769,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.9106000661849976,
-      "logits/rejected": -1.908151626586914,
-      "logps/chosen": -31.15958023071289,
-      "logps/rejected": -30.377086639404297,
-      "loss": 0.419,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.30450108647346497,
-      "rewards/margins": 0.42469558119773865,
-      "rewards/rejected": -0.12019453942775726,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.9983298778533936,
-      "logits/rejected": -1.9911689758300781,
-      "logps/chosen": -33.74740219116211,
-      "logps/rejected": -33.24574279785156,
-      "loss": 0.3833,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.29125431180000305,
-      "rewards/margins": 0.4282079339027405,
-      "rewards/rejected": -0.13695362210273743,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.9616893529891968,
-      "logits/rejected": -1.9681020975112915,
-      "logps/chosen": -31.261194229125977,
-      "logps/rejected": -32.988670349121094,
-      "loss": 0.29,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.29519280791282654,
-      "rewards/margins": 0.43315425515174866,
-      "rewards/rejected": -0.1379614621400833,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -2.0316014289855957,
-      "logits/rejected": -2.0083844661712646,
-      "logps/chosen": -30.99959945678711,
-      "logps/rejected": -33.247528076171875,
-      "loss": 0.3983,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.2627808749675751,
-      "rewards/margins": 0.4136640131473541,
-      "rewards/rejected": -0.15088310837745667,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.9671332836151123,
-      "logits/rejected": -1.9622156620025635,
-      "logps/chosen": -31.91083335876465,
-      "logps/rejected": -32.353614807128906,
-      "loss": 0.4301,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.3035394251346588,
-      "rewards/margins": 0.4431583881378174,
-      "rewards/rejected": -0.13961896300315857,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -2.01167368888855,
-      "logits/rejected": -2.0135717391967773,
-      "logps/chosen": -34.01561737060547,
-      "logps/rejected": -35.08711624145508,
-      "loss": 0.36,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.21610423922538757,
-      "rewards/margins": 0.38782355189323425,
-      "rewards/rejected": -0.17171931266784668,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.9978135824203491,
-      "logits/rejected": -1.9970858097076416,
-      "logps/chosen": -30.205724716186523,
-      "logps/rejected": -31.725692749023438,
-      "loss": 0.3897,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.23179996013641357,
-      "rewards/margins": 0.3902333974838257,
-      "rewards/rejected": -0.1584334373474121,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -2.0145509243011475,
-      "logits/rejected": -2.012350082397461,
-      "logps/chosen": -32.4493408203125,
-      "logps/rejected": -32.55584716796875,
-      "loss": 0.5531,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3293648362159729,
-      "rewards/margins": 0.4828970432281494,
-      "rewards/rejected": -0.15353217720985413,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.2464921474456787,
-      "eval_logits/rejected": -2.241631031036377,
-      "eval_logps/chosen": -34.027915954589844,
-      "eval_logps/rejected": -37.547115325927734,
-      "eval_loss": 0.7434703707695007,
-      "eval_rewards/accuracies": 0.5577242374420166,
-      "eval_rewards/chosen": 0.003980191424489021,
-      "eval_rewards/margins": 0.022275729104876518,
-      "eval_rewards/rejected": -0.018295539543032646,
-      "eval_runtime": 145.9008,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -2.0622594356536865,
-      "logits/rejected": -2.0689759254455566,
-      "logps/chosen": -31.731964111328125,
-      "logps/rejected": -32.2499885559082,
-      "loss": 0.2891,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.3032490909099579,
-      "rewards/margins": 0.4153287410736084,
-      "rewards/rejected": -0.11207965761423111,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -2.0177249908447266,
-      "logits/rejected": -2.0152080059051514,
-      "logps/chosen": -34.171417236328125,
-      "logps/rejected": -31.206134796142578,
-      "loss": 0.5415,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.35067808628082275,
-      "rewards/margins": 0.4830544888973236,
-      "rewards/rejected": -0.13237646222114563,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.9650866985321045,
-      "logits/rejected": -1.9581254720687866,
-      "logps/chosen": -34.88035202026367,
-      "logps/rejected": -32.54679489135742,
-      "loss": 0.4357,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.35032278299331665,
-      "rewards/margins": 0.4640774130821228,
-      "rewards/rejected": -0.11375461518764496,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.9704965353012085,
-      "logits/rejected": -1.9766871929168701,
-      "logps/chosen": -33.551597595214844,
-      "logps/rejected": -33.46538162231445,
-      "loss": 0.2942,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.2566205859184265,
-      "rewards/margins": 0.39169201254844666,
-      "rewards/rejected": -0.13507142663002014,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -2.0592033863067627,
-      "logits/rejected": -2.073496103286743,
-      "logps/chosen": -31.369670867919922,
-      "logps/rejected": -32.21284484863281,
-      "loss": 0.3661,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.28967565298080444,
-      "rewards/margins": 0.4187701642513275,
-      "rewards/rejected": -0.12909449636936188,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.9380521774291992,
-      "logits/rejected": -1.9349801540374756,
-      "logps/chosen": -33.320587158203125,
-      "logps/rejected": -34.86935806274414,
-      "loss": 0.5616,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3614245057106018,
-      "rewards/margins": 0.46288686990737915,
-      "rewards/rejected": -0.10146232694387436,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -2.0301218032836914,
-      "logits/rejected": -2.0302772521972656,
-      "logps/chosen": -30.811248779296875,
-      "logps/rejected": -34.03461837768555,
-      "loss": 0.2823,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.2846068739891052,
-      "rewards/margins": 0.40955057740211487,
-      "rewards/rejected": -0.12494371086359024,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.97750723361969,
-      "logits/rejected": -1.9745209217071533,
-      "logps/chosen": -30.4150333404541,
-      "logps/rejected": -33.014732360839844,
-      "loss": 0.4637,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.2743886411190033,
-      "rewards/margins": 0.3925444185733795,
-      "rewards/rejected": -0.11815576255321503,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -2.015723466873169,
-      "logits/rejected": -2.0155797004699707,
-      "logps/chosen": -29.640954971313477,
-      "logps/rejected": -30.86568832397461,
-      "loss": 0.3834,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.31328272819519043,
-      "rewards/margins": 0.4303969740867615,
-      "rewards/rejected": -0.11711428314447403,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.9227421283721924,
-      "logits/rejected": -1.9260936975479126,
-      "logps/chosen": -34.0415153503418,
-      "logps/rejected": -31.62136459350586,
-      "loss": 0.4315,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.2621740996837616,
-      "rewards/margins": 0.3561774790287018,
-      "rewards/rejected": -0.09400339424610138,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.2498111724853516,
-      "eval_logits/rejected": -2.2449464797973633,
-      "eval_logps/chosen": -34.047359466552734,
-      "eval_logps/rejected": -37.52130126953125,
-      "eval_loss": 0.8140609264373779,
-      "eval_rewards/accuracies": 0.49958473443984985,
-      "eval_rewards/chosen": -0.007684962823987007,
-      "eval_rewards/margins": -0.0048751430585980415,
-      "eval_rewards/rejected": -0.0028098197653889656,
-      "eval_runtime": 145.859,
-      "eval_samples_per_second": 2.352,
-      "eval_steps_per_second": 0.295,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -2.0064587593078613,
-      "logits/rejected": -1.9971309900283813,
-      "logps/chosen": -36.031028747558594,
-      "logps/rejected": -32.0997428894043,
-      "loss": 0.2508,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.28859394788742065,
-      "rewards/margins": 0.4049535393714905,
-      "rewards/rejected": -0.11635950952768326,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -2.020625352859497,
-      "logits/rejected": -2.0215954780578613,
-      "logps/chosen": -35.916099548339844,
-      "logps/rejected": -33.29255676269531,
-      "loss": 0.416,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.31157156825065613,
-      "rewards/margins": 0.4414919316768646,
-      "rewards/rejected": -0.1299203783273697,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -2.0350329875946045,
-      "logits/rejected": -2.032480239868164,
-      "logps/chosen": -32.11791229248047,
-      "logps/rejected": -33.21588897705078,
-      "loss": 0.3069,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.2672501802444458,
-      "rewards/margins": 0.3986472487449646,
-      "rewards/rejected": -0.1313970983028412,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -2.022247791290283,
-      "logits/rejected": -2.0273849964141846,
-      "logps/chosen": -31.30985450744629,
-      "logps/rejected": -33.424705505371094,
-      "loss": 0.3425,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.28552335500717163,
-      "rewards/margins": 0.447455495595932,
-      "rewards/rejected": -0.161932110786438,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.9659734964370728,
-      "logits/rejected": -1.9647700786590576,
-      "logps/chosen": -32.636070251464844,
-      "logps/rejected": -34.969459533691406,
-      "loss": 0.3445,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.2933460772037506,
-      "rewards/margins": 0.46344757080078125,
-      "rewards/rejected": -0.17010149359703064,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.8426628112792969,
-      "logits/rejected": -1.837442398071289,
-      "logps/chosen": -35.48710632324219,
-      "logps/rejected": -35.31467056274414,
-      "loss": 0.573,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3068006932735443,
-      "rewards/margins": 0.37055516242980957,
-      "rewards/rejected": -0.06375448405742645,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.983940839767456,
-      "logits/rejected": -1.9875280857086182,
-      "logps/chosen": -34.71583938598633,
-      "logps/rejected": -33.711788177490234,
-      "loss": 0.4316,
-      "rewards/accuracies": 0.8208333849906921,
-      "rewards/chosen": 0.29869017004966736,
-      "rewards/margins": 0.36885932087898254,
-      "rewards/rejected": -0.0701691135764122,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.9259727001190186,
-      "logits/rejected": -1.924078345298767,
-      "logps/chosen": -33.34611129760742,
-      "logps/rejected": -34.635719299316406,
-      "loss": 0.2786,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.33328744769096375,
-      "rewards/margins": 0.4719271659851074,
-      "rewards/rejected": -0.1386396586894989,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.973812460899353,
-      "logits/rejected": -1.9723854064941406,
-      "logps/chosen": -32.87940216064453,
-      "logps/rejected": -33.219146728515625,
-      "loss": 0.213,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.29780465364456177,
-      "rewards/margins": 0.37734922766685486,
-      "rewards/rejected": -0.0795445665717125,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.9056497812271118,
-      "logits/rejected": -1.899532675743103,
-      "logps/chosen": -33.77119064331055,
-      "logps/rejected": -31.565038681030273,
-      "loss": 0.3445,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3671306371688843,
-      "rewards/margins": 0.4275972247123718,
-      "rewards/rejected": -0.060466617345809937,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.248525857925415,
-      "eval_logits/rejected": -2.2436628341674805,
-      "eval_logps/chosen": -33.9709587097168,
-      "eval_logps/rejected": -37.455101013183594,
-      "eval_loss": 0.8104392886161804,
-      "eval_rewards/accuracies": 0.5282392501831055,
-      "eval_rewards/chosen": 0.038154613226652145,
-      "eval_rewards/margins": 0.0012450924841687083,
-      "eval_rewards/rejected": 0.036909520626068115,
-      "eval_runtime": 145.9117,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -2.0072484016418457,
-      "logits/rejected": -2.014772653579712,
-      "logps/chosen": -31.6469669342041,
-      "logps/rejected": -34.390533447265625,
-      "loss": 0.2774,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.2887730598449707,
-      "rewards/margins": 0.36958909034729004,
-      "rewards/rejected": -0.08081604540348053,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.9808098077774048,
-      "logits/rejected": -1.983655571937561,
-      "logps/chosen": -32.01128005981445,
-      "logps/rejected": -33.38429641723633,
-      "loss": 0.2516,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.33588114380836487,
-      "rewards/margins": 0.42857733368873596,
-      "rewards/rejected": -0.09269621223211288,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.8852159976959229,
-      "logits/rejected": -1.8893970251083374,
-      "logps/chosen": -32.007896423339844,
-      "logps/rejected": -35.13220977783203,
-      "loss": 0.2545,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.3501349091529846,
-      "rewards/margins": 0.4833507537841797,
-      "rewards/rejected": -0.13321581482887268,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.948075294494629,
-      "logits/rejected": -1.9441587924957275,
-      "logps/chosen": -32.47659683227539,
-      "logps/rejected": -33.118370056152344,
-      "loss": 0.2688,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.29523491859436035,
-      "rewards/margins": 0.3694687485694885,
-      "rewards/rejected": -0.07423384487628937,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -2.035362482070923,
-      "logits/rejected": -2.037264585494995,
-      "logps/chosen": -32.23796081542969,
-      "logps/rejected": -35.856971740722656,
-      "loss": 0.2484,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3224264085292816,
-      "rewards/margins": 0.4394490122795105,
-      "rewards/rejected": -0.11702261120080948,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -1.8381340503692627,
-      "logits/rejected": -1.8319995403289795,
-      "logps/chosen": -35.71126174926758,
-      "logps/rejected": -33.0915641784668,
-      "loss": 0.2965,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.34224915504455566,
-      "rewards/margins": 0.46204671263694763,
-      "rewards/rejected": -0.11979756504297256,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.9029734134674072,
-      "logits/rejected": -1.9030624628067017,
-      "logps/chosen": -35.23638916015625,
-      "logps/rejected": -34.9959602355957,
-      "loss": 0.1819,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3638507127761841,
-      "rewards/margins": 0.4536662995815277,
-      "rewards/rejected": -0.08981556445360184,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.9545879364013672,
-      "logits/rejected": -1.9431183338165283,
-      "logps/chosen": -33.588871002197266,
-      "logps/rejected": -33.196922302246094,
-      "loss": 0.2214,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.31062597036361694,
-      "rewards/margins": 0.4755612015724182,
-      "rewards/rejected": -0.16493523120880127,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -2.055995464324951,
-      "logits/rejected": -2.0605499744415283,
-      "logps/chosen": -32.802162170410156,
-      "logps/rejected": -32.60199737548828,
-      "loss": 0.3724,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3166634738445282,
-      "rewards/margins": 0.41597333550453186,
-      "rewards/rejected": -0.09930987656116486,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.9720876216888428,
-      "logits/rejected": -1.9648761749267578,
-      "logps/chosen": -34.31425857543945,
-      "logps/rejected": -32.596370697021484,
-      "loss": 0.1902,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3238928020000458,
-      "rewards/margins": 0.44043517112731934,
-      "rewards/rejected": -0.11654232442378998,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.2474730014801025,
-      "eval_logits/rejected": -2.24257755279541,
-      "eval_logps/chosen": -33.979740142822266,
-      "eval_logps/rejected": -37.45598220825195,
-      "eval_loss": 0.8122833371162415,
-      "eval_rewards/accuracies": 0.5199335813522339,
-      "eval_rewards/chosen": 0.03288789838552475,
-      "eval_rewards/margins": -0.003495381912216544,
-      "eval_rewards/rejected": 0.03638327494263649,
-      "eval_runtime": 145.9163,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.9646570682525635,
-      "logits/rejected": -1.9652878046035767,
-      "logps/chosen": -30.66654396057129,
-      "logps/rejected": -34.830745697021484,
-      "loss": 0.3036,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.30036959052085876,
-      "rewards/margins": 0.38500353693962097,
-      "rewards/rejected": -0.08463393151760101,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.9272854328155518,
-      "logits/rejected": -1.9191501140594482,
-      "logps/chosen": -31.269882202148438,
-      "logps/rejected": -33.781036376953125,
-      "loss": 0.2861,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3414386212825775,
-      "rewards/margins": 0.46723994612693787,
-      "rewards/rejected": -0.12580129504203796,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -2.09513521194458,
-      "logits/rejected": -2.095780849456787,
-      "logps/chosen": -29.461566925048828,
-      "logps/rejected": -33.07443618774414,
-      "loss": 0.3863,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3212849497795105,
-      "rewards/margins": 0.4470299780368805,
-      "rewards/rejected": -0.12574505805969238,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.9750986099243164,
-      "logits/rejected": -1.984668493270874,
-      "logps/chosen": -33.88821792602539,
-      "logps/rejected": -33.15973663330078,
-      "loss": 0.229,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3221094012260437,
-      "rewards/margins": 0.4241645336151123,
-      "rewards/rejected": -0.10205511748790741,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -2.016716480255127,
-      "logits/rejected": -2.0206027030944824,
-      "logps/chosen": -30.1220703125,
-      "logps/rejected": -31.91337013244629,
-      "loss": 0.2308,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.31539037823677063,
-      "rewards/margins": 0.42176294326782227,
-      "rewards/rejected": -0.10637259483337402,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -2.017374038696289,
-      "logits/rejected": -2.0133793354034424,
-      "logps/chosen": -33.43627166748047,
-      "logps/rejected": -33.95038986206055,
-      "loss": 0.246,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.38336849212646484,
-      "rewards/margins": 0.4097708761692047,
-      "rewards/rejected": -0.026402413845062256,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.9316686391830444,
-      "logits/rejected": -1.9342737197875977,
-      "logps/chosen": -32.216575622558594,
-      "logps/rejected": -35.39495849609375,
-      "loss": 0.2262,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.35770565271377563,
-      "rewards/margins": 0.4857249855995178,
-      "rewards/rejected": -0.1280193030834198,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.9653689861297607,
-      "logits/rejected": -1.958824872970581,
-      "logps/chosen": -34.86272048950195,
-      "logps/rejected": -36.444908142089844,
-      "loss": 0.4198,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.32540959119796753,
-      "rewards/margins": 0.43518051505088806,
-      "rewards/rejected": -0.10977087914943695,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.9367698431015015,
-      "logits/rejected": -1.9314988851547241,
-      "logps/chosen": -30.504741668701172,
-      "logps/rejected": -36.1971549987793,
-      "loss": 0.2745,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.29892516136169434,
-      "rewards/margins": 0.4232874810695648,
-      "rewards/rejected": -0.12436232715845108,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.9119179248809814,
-      "logits/rejected": -1.9008598327636719,
-      "logps/chosen": -31.161075592041016,
-      "logps/rejected": -29.76761245727539,
-      "loss": 0.1724,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.35256943106651306,
-      "rewards/margins": 0.45329660177230835,
-      "rewards/rejected": -0.10072717815637589,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.2481319904327393,
-      "eval_logits/rejected": -2.243258237838745,
-      "eval_logps/chosen": -33.95476150512695,
-      "eval_logps/rejected": -37.45245361328125,
-      "eval_loss": 0.7849614024162292,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": 0.04787486046552658,
-      "eval_rewards/margins": 0.009373747743666172,
-      "eval_rewards/rejected": 0.03850111365318298,
-      "eval_runtime": 145.98,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -2.025754928588867,
-      "logits/rejected": -2.0250885486602783,
-      "logps/chosen": -31.275997161865234,
-      "logps/rejected": -30.338693618774414,
-      "loss": 0.1764,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3289177417755127,
-      "rewards/margins": 0.3770465552806854,
-      "rewards/rejected": -0.048128847032785416,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -2.061776876449585,
-      "logits/rejected": -2.0624749660491943,
-      "logps/chosen": -31.365009307861328,
-      "logps/rejected": -31.31036949157715,
-      "loss": 0.2275,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.34714099764823914,
-      "rewards/margins": 0.43387675285339355,
-      "rewards/rejected": -0.08673573285341263,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -2.035461664199829,
-      "logits/rejected": -2.038501024246216,
-      "logps/chosen": -31.733938217163086,
-      "logps/rejected": -28.1260929107666,
-      "loss": 0.2038,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.327678382396698,
-      "rewards/margins": 0.4131060540676117,
-      "rewards/rejected": -0.08542772382497787,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -2.0482096672058105,
-      "logits/rejected": -2.0403339862823486,
-      "logps/chosen": -30.584192276000977,
-      "logps/rejected": -33.65731430053711,
-      "loss": 0.2815,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3562210202217102,
-      "rewards/margins": 0.4504729211330414,
-      "rewards/rejected": -0.09425191581249237,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.975842833518982,
-      "logits/rejected": -1.986856460571289,
-      "logps/chosen": -33.01598358154297,
-      "logps/rejected": -32.32152557373047,
-      "loss": 0.2538,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.32227379083633423,
-      "rewards/margins": 0.3681793212890625,
-      "rewards/rejected": -0.04590549319982529,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -2.056307077407837,
-      "logits/rejected": -2.0468807220458984,
-      "logps/chosen": -32.9362907409668,
-      "logps/rejected": -36.47593688964844,
-      "loss": 0.4119,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3386926054954529,
-      "rewards/margins": 0.4254216253757477,
-      "rewards/rejected": -0.0867290273308754,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.9632923603057861,
-      "logits/rejected": -1.9650466442108154,
-      "logps/chosen": -35.4192008972168,
-      "logps/rejected": -32.78428268432617,
-      "loss": 0.2636,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3420516550540924,
-      "rewards/margins": 0.3938991129398346,
-      "rewards/rejected": -0.05184745788574219,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -2.100684404373169,
-      "logits/rejected": -2.101898193359375,
-      "logps/chosen": -32.51726150512695,
-      "logps/rejected": -34.33898162841797,
-      "loss": 0.24,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.32422804832458496,
-      "rewards/margins": 0.4582032561302185,
-      "rewards/rejected": -0.1339752972126007,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -2.06551194190979,
-      "logits/rejected": -2.0696327686309814,
-      "logps/chosen": -30.335834503173828,
-      "logps/rejected": -33.68609619140625,
-      "loss": 0.2635,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3344890773296356,
-      "rewards/margins": 0.41744667291641235,
-      "rewards/rejected": -0.08295761048793793,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -2.05393385887146,
-      "logits/rejected": -2.059272289276123,
-      "logps/chosen": -33.025550842285156,
-      "logps/rejected": -30.527883529663086,
-      "loss": 0.1864,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3260694444179535,
-      "rewards/margins": 0.4017953872680664,
-      "rewards/rejected": -0.07572595030069351,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.244070529937744,
-      "eval_logits/rejected": -2.2392044067382812,
-      "eval_logps/chosen": -33.96210861206055,
-      "eval_logps/rejected": -37.46709060668945,
-      "eval_loss": 0.7625167369842529,
-      "eval_rewards/accuracies": 0.5132890343666077,
-      "eval_rewards/chosen": 0.04346639662981033,
-      "eval_rewards/margins": 0.013746440410614014,
-      "eval_rewards/rejected": 0.02971995621919632,
-      "eval_runtime": 145.9267,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -2.046633243560791,
-      "logits/rejected": -2.046231269836426,
-      "logps/chosen": -32.30084991455078,
-      "logps/rejected": -31.1953125,
-      "loss": 0.3011,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3835814893245697,
-      "rewards/margins": 0.491219699382782,
-      "rewards/rejected": -0.10763820260763168,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.8906314373016357,
-      "logits/rejected": -1.8921263217926025,
-      "logps/chosen": -34.062103271484375,
-      "logps/rejected": -33.113182067871094,
-      "loss": 0.3593,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.30172425508499146,
-      "rewards/margins": 0.34487384557724,
-      "rewards/rejected": -0.043149612843990326,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.9306659698486328,
-      "logits/rejected": -1.92781662940979,
-      "logps/chosen": -30.51358985900879,
-      "logps/rejected": -31.30172348022461,
-      "loss": 0.3529,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.29887938499450684,
-      "rewards/margins": 0.3804691433906555,
-      "rewards/rejected": -0.08158973604440689,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -2.0672850608825684,
-      "logits/rejected": -2.061563730239868,
-      "logps/chosen": -32.45098876953125,
-      "logps/rejected": -33.04710006713867,
-      "loss": 0.2299,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3532842993736267,
-      "rewards/margins": 0.40913909673690796,
-      "rewards/rejected": -0.05585480481386185,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.9143331050872803,
-      "logits/rejected": -1.9041531085968018,
-      "logps/chosen": -32.86902618408203,
-      "logps/rejected": -31.468719482421875,
-      "loss": 0.2081,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.349628746509552,
-      "rewards/margins": 0.455420583486557,
-      "rewards/rejected": -0.1057918444275856,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.9975311756134033,
-      "logits/rejected": -1.9972455501556396,
-      "logps/chosen": -33.09718704223633,
-      "logps/rejected": -31.396026611328125,
-      "loss": 0.1634,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3507378101348877,
-      "rewards/margins": 0.4674840569496155,
-      "rewards/rejected": -0.11674626171588898,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -2.0674796104431152,
-      "logits/rejected": -2.0618603229522705,
-      "logps/chosen": -32.99332809448242,
-      "logps/rejected": -33.066471099853516,
-      "loss": 0.1455,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3195871412754059,
-      "rewards/margins": 0.3947373330593109,
-      "rewards/rejected": -0.07515018433332443,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.9187389612197876,
-      "logits/rejected": -1.9265291690826416,
-      "logps/chosen": -32.774940490722656,
-      "logps/rejected": -32.578086853027344,
-      "loss": 0.1261,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.334833025932312,
-      "rewards/margins": 0.46482500433921814,
-      "rewards/rejected": -0.12999196350574493,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -2.0002593994140625,
-      "logits/rejected": -1.9944689273834229,
-      "logps/chosen": -31.51430892944336,
-      "logps/rejected": -33.402320861816406,
-      "loss": 0.1302,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3955826163291931,
-      "rewards/margins": 0.46690741181373596,
-      "rewards/rejected": -0.07132478058338165,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.9393714666366577,
-      "logits/rejected": -1.9415557384490967,
-      "logps/chosen": -32.937740325927734,
-      "logps/rejected": -32.9244499206543,
-      "loss": 0.1629,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3316211700439453,
-      "rewards/margins": 0.4302603602409363,
-      "rewards/rejected": -0.09863914549350739,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.2436180114746094,
-      "eval_logits/rejected": -2.238736629486084,
-      "eval_logps/chosen": -33.95472717285156,
-      "eval_logps/rejected": -37.46039581298828,
-      "eval_loss": 0.7623819708824158,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.04789462685585022,
-      "eval_rewards/margins": 0.01415975671261549,
-      "eval_rewards/rejected": 0.033734869211912155,
-      "eval_runtime": 145.9396,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.8697309494018555,
-      "logits/rejected": -1.8685888051986694,
-      "logps/chosen": -35.43067169189453,
-      "logps/rejected": -32.846221923828125,
-      "loss": 0.186,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.3730871081352234,
-      "rewards/margins": 0.4552043080329895,
-      "rewards/rejected": -0.0821172297000885,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -2.0492749214172363,
-      "logits/rejected": -2.043208122253418,
-      "logps/chosen": -31.149316787719727,
-      "logps/rejected": -33.047523498535156,
-      "loss": 0.147,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.33262285590171814,
-      "rewards/margins": 0.43401265144348145,
-      "rewards/rejected": -0.10138984769582748,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.8950700759887695,
-      "logits/rejected": -1.8832353353500366,
-      "logps/chosen": -33.81637191772461,
-      "logps/rejected": -33.06397247314453,
-      "loss": 0.1532,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3415863811969757,
-      "rewards/margins": 0.4436774253845215,
-      "rewards/rejected": -0.10209109634160995,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.9864425659179688,
-      "logits/rejected": -1.979435682296753,
-      "logps/chosen": -30.0882625579834,
-      "logps/rejected": -29.048751831054688,
-      "loss": 0.1166,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.3671828806400299,
-      "rewards/margins": 0.4758971631526947,
-      "rewards/rejected": -0.10871431976556778,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -2.020458221435547,
-      "logits/rejected": -2.026707172393799,
-      "logps/chosen": -34.78377914428711,
-      "logps/rejected": -31.65704917907715,
-      "loss": 0.1432,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.36696386337280273,
-      "rewards/margins": 0.4502052664756775,
-      "rewards/rejected": -0.08324144035577774,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.9884742498397827,
-      "logits/rejected": -1.9890835285186768,
-      "logps/chosen": -32.45567321777344,
-      "logps/rejected": -32.96289825439453,
-      "loss": 0.1291,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3437303900718689,
-      "rewards/margins": 0.43254727125167847,
-      "rewards/rejected": -0.08881691098213196,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.9342743158340454,
-      "logits/rejected": -1.9337249994277954,
-      "logps/chosen": -32.51142501831055,
-      "logps/rejected": -33.648887634277344,
-      "loss": 0.1225,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.34764808416366577,
-      "rewards/margins": 0.42461419105529785,
-      "rewards/rejected": -0.07696612179279327,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.8161872625350952,
-      "logits/rejected": -1.8113391399383545,
-      "logps/chosen": -34.59672164916992,
-      "logps/rejected": -31.089731216430664,
-      "loss": 0.1114,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.3844338059425354,
-      "rewards/margins": 0.48237746953964233,
-      "rewards/rejected": -0.09794367849826813,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -2.023128032684326,
-      "logits/rejected": -2.007215976715088,
-      "logps/chosen": -33.77648162841797,
-      "logps/rejected": -35.00053787231445,
-      "loss": 0.0953,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.34265583753585815,
-      "rewards/margins": 0.4837895929813385,
-      "rewards/rejected": -0.14113374054431915,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.9649696350097656,
-      "logits/rejected": -1.969999074935913,
-      "logps/chosen": -32.89286804199219,
-      "logps/rejected": -36.32848358154297,
-      "loss": 0.1881,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.37485235929489136,
-      "rewards/margins": 0.5100094676017761,
-      "rewards/rejected": -0.13515713810920715,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.243983507156372,
-      "eval_logits/rejected": -2.2391045093536377,
-      "eval_logps/chosen": -33.958831787109375,
-      "eval_logps/rejected": -37.44320297241211,
-      "eval_loss": 0.7945417165756226,
-      "eval_rewards/accuracies": 0.4966777563095093,
-      "eval_rewards/chosen": 0.045431531965732574,
-      "eval_rewards/margins": 0.0013837195001542568,
-      "eval_rewards/rejected": 0.04404781758785248,
-      "eval_runtime": 145.9089,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.9365456104278564,
-      "logits/rejected": -1.9366706609725952,
-      "logps/chosen": -35.67008590698242,
-      "logps/rejected": -37.238311767578125,
-      "loss": 0.1675,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.3443610668182373,
-      "rewards/margins": 0.4368794560432434,
-      "rewards/rejected": -0.09251832962036133,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -2.0239405632019043,
-      "logits/rejected": -2.022326946258545,
-      "logps/chosen": -31.496356964111328,
-      "logps/rejected": -31.08811378479004,
-      "loss": 0.1558,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.35850051045417786,
-      "rewards/margins": 0.43935075402259827,
-      "rewards/rejected": -0.08085022866725922,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.9284435510635376,
-      "logits/rejected": -1.9298274517059326,
-      "logps/chosen": -33.75816345214844,
-      "logps/rejected": -33.829715728759766,
-      "loss": 0.127,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.36965471506118774,
-      "rewards/margins": 0.47827833890914917,
-      "rewards/rejected": -0.10862363874912262,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.9887325763702393,
-      "logits/rejected": -1.9875112771987915,
-      "logps/chosen": -30.637014389038086,
-      "logps/rejected": -32.18603515625,
-      "loss": 0.1273,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.33699196577072144,
-      "rewards/margins": 0.4534294009208679,
-      "rewards/rejected": -0.11643745750188828,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -2.005549907684326,
-      "logits/rejected": -1.9907426834106445,
-      "logps/chosen": -33.7475700378418,
-      "logps/rejected": -36.4091796875,
-      "loss": 0.1417,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3414615988731384,
-      "rewards/margins": 0.4283091127872467,
-      "rewards/rejected": -0.08684752136468887,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -2.0126748085021973,
-      "logits/rejected": -2.0252599716186523,
-      "logps/chosen": -31.497417449951172,
-      "logps/rejected": -32.39682388305664,
-      "loss": 0.1287,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3426300883293152,
-      "rewards/margins": 0.4368777275085449,
-      "rewards/rejected": -0.09424759447574615,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -2.0992069244384766,
-      "logits/rejected": -2.0984139442443848,
-      "logps/chosen": -31.42146873474121,
-      "logps/rejected": -36.053802490234375,
-      "loss": 0.146,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.36537685990333557,
-      "rewards/margins": 0.49912720918655396,
-      "rewards/rejected": -0.13375037908554077,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.9654130935668945,
-      "logits/rejected": -1.9733936786651611,
-      "logps/chosen": -33.638160705566406,
-      "logps/rejected": -37.61304473876953,
-      "loss": 0.1702,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.325391948223114,
-      "rewards/margins": 0.423430860042572,
-      "rewards/rejected": -0.0980389267206192,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.9179338216781616,
-      "logits/rejected": -1.913713812828064,
-      "logps/chosen": -34.152076721191406,
-      "logps/rejected": -31.679922103881836,
-      "loss": 0.1542,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3690822720527649,
-      "rewards/margins": 0.4186008870601654,
-      "rewards/rejected": -0.049518562853336334,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -2.0843544006347656,
-      "logits/rejected": -2.0817577838897705,
-      "logps/chosen": -30.708471298217773,
-      "logps/rejected": -32.34890365600586,
-      "loss": 0.1395,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.3528476655483246,
-      "rewards/margins": 0.47550955414772034,
-      "rewards/rejected": -0.12266188859939575,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.2457053661346436,
-      "eval_logits/rejected": -2.240827798843384,
-      "eval_logps/chosen": -33.95180130004883,
-      "eval_logps/rejected": -37.45983123779297,
-      "eval_loss": 0.7520378232002258,
-      "eval_rewards/accuracies": 0.545265793800354,
-      "eval_rewards/chosen": 0.04964962601661682,
-      "eval_rewards/margins": 0.015574713237583637,
-      "eval_rewards/rejected": 0.03407491371035576,
-      "eval_runtime": 145.8805,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -2.0233004093170166,
-      "logits/rejected": -2.0188450813293457,
-      "logps/chosen": -33.010955810546875,
-      "logps/rejected": -29.703121185302734,
-      "loss": 0.152,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.3789446949958801,
-      "rewards/margins": 0.46814146637916565,
-      "rewards/rejected": -0.08919680118560791,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.9822219610214233,
-      "logits/rejected": -1.9834082126617432,
-      "logps/chosen": -32.7589111328125,
-      "logps/rejected": -35.04393768310547,
-      "loss": 0.1561,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.36357951164245605,
-      "rewards/margins": 0.4355399012565613,
-      "rewards/rejected": -0.07196036726236343,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.9348289966583252,
-      "logits/rejected": -1.9279505014419556,
-      "logps/chosen": -33.61431121826172,
-      "logps/rejected": -36.830039978027344,
-      "loss": 0.1341,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.3164740204811096,
-      "rewards/margins": 0.40085238218307495,
-      "rewards/rejected": -0.08437839895486832,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.9475971460342407,
-      "logits/rejected": -1.9420528411865234,
-      "logps/chosen": -28.74324607849121,
-      "logps/rejected": -31.90035057067871,
-      "loss": 0.1889,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3419957458972931,
-      "rewards/margins": 0.44381189346313477,
-      "rewards/rejected": -0.10181611776351929,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.9640169143676758,
-      "logits/rejected": -1.9535976648330688,
-      "logps/chosen": -31.90164566040039,
-      "logps/rejected": -35.4890251159668,
-      "loss": 0.1843,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.34793075919151306,
-      "rewards/margins": 0.47012752294540405,
-      "rewards/rejected": -0.12219683080911636,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.9425592422485352,
-      "logits/rejected": -1.9395831823349,
-      "logps/chosen": -32.72911834716797,
-      "logps/rejected": -31.02676773071289,
-      "loss": 0.1479,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3483937978744507,
-      "rewards/margins": 0.4299589991569519,
-      "rewards/rejected": -0.08156517893075943,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -2.0044729709625244,
-      "logits/rejected": -2.011080741882324,
-      "logps/chosen": -31.44146156311035,
-      "logps/rejected": -33.138694763183594,
-      "loss": 0.1786,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.37662890553474426,
-      "rewards/margins": 0.48239096999168396,
-      "rewards/rejected": -0.10576207935810089,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -2.0032641887664795,
-      "logits/rejected": -2.0138297080993652,
-      "logps/chosen": -31.077510833740234,
-      "logps/rejected": -29.825885772705078,
-      "loss": 0.1168,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.3706461489200592,
-      "rewards/margins": 0.48903828859329224,
-      "rewards/rejected": -0.11839208751916885,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.9371792078018188,
-      "logits/rejected": -1.9299945831298828,
-      "logps/chosen": -33.08808898925781,
-      "logps/rejected": -32.423980712890625,
-      "loss": 0.1489,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.3410363495349884,
-      "rewards/margins": 0.4691395163536072,
-      "rewards/rejected": -0.12810321152210236,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.987765908241272,
-      "logits/rejected": -1.9844974279403687,
-      "logps/chosen": -33.07574462890625,
-      "logps/rejected": -33.45602035522461,
-      "loss": 0.1436,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.32989656925201416,
-      "rewards/margins": 0.44737356901168823,
-      "rewards/rejected": -0.11747703701257706,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.245612859725952,
-      "eval_logits/rejected": -2.2407407760620117,
-      "eval_logps/chosen": -33.94506072998047,
-      "eval_logps/rejected": -37.460636138916016,
-      "eval_loss": 0.7351647019386292,
-      "eval_rewards/accuracies": 0.5398671627044678,
-      "eval_rewards/chosen": 0.05369636043906212,
-      "eval_rewards/margins": 0.02010684460401535,
-      "eval_rewards/rejected": 0.03358951583504677,
-      "eval_runtime": 145.9788,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -2.067953586578369,
-      "logits/rejected": -2.068563461303711,
-      "logps/chosen": -29.285715103149414,
-      "logps/rejected": -32.92698669433594,
-      "loss": 0.2375,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3183347284793854,
-      "rewards/margins": 0.43025675415992737,
-      "rewards/rejected": -0.11192206293344498,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.9213430881500244,
-      "logits/rejected": -1.9212009906768799,
-      "logps/chosen": -32.53323745727539,
-      "logps/rejected": -34.081382751464844,
-      "loss": 0.1987,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3051798939704895,
-      "rewards/margins": 0.42977064847946167,
-      "rewards/rejected": -0.12459073960781097,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -2.0738425254821777,
-      "logits/rejected": -2.071150541305542,
-      "logps/chosen": -30.976924896240234,
-      "logps/rejected": -31.667194366455078,
-      "loss": 0.1555,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.2982819676399231,
-      "rewards/margins": 0.4084067940711975,
-      "rewards/rejected": -0.11012481153011322,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "learning_rate": 0.0,
-      "logits/chosen": -2.056631326675415,
-      "logits/rejected": -2.0577590465545654,
-      "logps/chosen": -30.05573081970215,
-      "logps/rejected": -30.0135498046875,
-      "loss": 0.1485,
-      "rewards/accuracies": 0.970833420753479,
-      "rewards/chosen": 0.33852633833885193,
-      "rewards/margins": 0.47254690527915955,
-      "rewards/rejected": -0.13402055203914642,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.2286405209984098,
-      "train_runtime": 10803.5829,
-      "train_samples_per_second": 1.14,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.859264765157328,
+      "train_runtime": 3251.7628,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,