diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2537 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8668904304504395,
-      "logits/rejected": -1.8712035417556763,
-      "logps/chosen": -36.981239318847656,
-      "logps/rejected": -33.63866424560547,
-      "loss": 0.6837,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.017751876264810562,
-      "rewards/margins": 0.023763436824083328,
-      "rewards/rejected": -0.0060115596279501915,
+      "logits/chosen": -1.8667322397232056,
+      "logits/rejected": -1.8710299730300903,
+      "logps/chosen": -36.98433303833008,
+      "logps/rejected": -33.65640640258789,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.015585740096867085,
+      "rewards/margins": 0.034019216895103455,
+      "rewards/rejected": -0.018433474004268646,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9975477457046509,
-      "logits/rejected": -2.0001885890960693,
-      "logps/chosen": -29.638402938842773,
-      "logps/rejected": -29.045080184936523,
-      "loss": 0.7005,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.002654150128364563,
-      "rewards/margins": -0.010201702825725079,
-      "rewards/rejected": 0.012855852022767067,
+      "logits/chosen": -1.9972655773162842,
+      "logits/rejected": -1.9999065399169922,
+      "logps/chosen": -29.6416072845459,
+      "logps/rejected": -29.046981811523438,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00041136163054034114,
+      "rewards/margins": -0.011113673448562622,
+      "rewards/rejected": 0.011525033973157406,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9209153652191162,
-      "logits/rejected": -1.9182332754135132,
-      "logps/chosen": -31.392269134521484,
-      "logps/rejected": -33.214996337890625,
-      "loss": 0.6893,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.016679534688591957,
-      "rewards/margins": 0.014253495261073112,
-      "rewards/rejected": 0.0024260382633656263,
+      "logits/chosen": -1.9205191135406494,
+      "logits/rejected": -1.9178133010864258,
+      "logps/chosen": -31.385412216186523,
+      "logps/rejected": -33.23365020751953,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.02147640287876129,
+      "rewards/margins": 0.03211076930165291,
+      "rewards/rejected": -0.010634368285536766,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017472505569458,
-      "logits/rejected": -2.0087223052978516,
-      "logps/chosen": -32.56648635864258,
-      "logps/rejected": -32.53681564331055,
-      "loss": 0.6837,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.007227012421935797,
-      "rewards/margins": 0.02412785217165947,
-      "rewards/rejected": -0.016900835558772087,
+      "logits/chosen": -2.017784833908081,
+      "logits/rejected": -2.009059429168701,
+      "logps/chosen": -32.547088623046875,
+      "logps/rejected": -32.473243713378906,
+      "loss": 0.5024,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.020800117403268814,
+      "rewards/margins": -0.006797379348427057,
+      "rewards/rejected": 0.027597496286034584,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8619842529296875,
-      "logits/rejected": -1.8512147665023804,
-      "logps/chosen": -33.57755661010742,
-      "logps/rejected": -35.45317840576172,
-      "loss": 0.7035,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.014055396430194378,
-      "rewards/margins": -0.013470378704369068,
-      "rewards/rejected": -0.0005850124871358275,
+      "logits/chosen": -1.863996148109436,
+      "logits/rejected": -1.8532311916351318,
+      "logps/chosen": -33.490657806396484,
+      "logps/rejected": -35.415367126464844,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.046776313334703445,
+      "rewards/margins": 0.02089320495724678,
+      "rewards/rejected": 0.025883108377456665,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.940241813659668,
-      "logits/rejected": -1.942185401916504,
-      "logps/chosen": -32.55712127685547,
-      "logps/rejected": -33.23926544189453,
-      "loss": 0.6629,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.030143504962325096,
-      "rewards/margins": 0.07766537368297577,
-      "rewards/rejected": -0.047521863132715225,
+      "logits/chosen": -1.9439821243286133,
+      "logits/rejected": -1.9459091424942017,
+      "logps/chosen": -32.50756072998047,
+      "logps/rejected": -33.15720748901367,
+      "loss": 0.489,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.06483479589223862,
+      "rewards/margins": 0.05491489917039871,
+      "rewards/rejected": 0.009919902309775352,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.070986270904541,
-      "logits/rejected": -2.0759458541870117,
-      "logps/chosen": -33.997291564941406,
-      "logps/rejected": -36.65125274658203,
-      "loss": 0.6775,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.006164415739476681,
-      "rewards/margins": 0.051500022411346436,
-      "rewards/rejected": -0.057664431631565094,
+      "logits/chosen": -2.075462579727173,
+      "logits/rejected": -2.0804481506347656,
+      "logps/chosen": -33.86588668823242,
+      "logps/rejected": -36.501365661621094,
+      "loss": 0.491,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.08581922948360443,
+      "rewards/margins": 0.038563162088394165,
+      "rewards/rejected": 0.04725607484579086,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9328521490097046,
-      "logits/rejected": -1.935974359512329,
-      "logps/chosen": -34.317806243896484,
-      "logps/rejected": -34.648555755615234,
-      "loss": 0.6465,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.07700104266405106,
-      "rewards/margins": 0.11630574613809586,
-      "rewards/rejected": -0.0393047034740448,
+      "logits/chosen": -1.9374949932098389,
+      "logits/rejected": -1.9405946731567383,
+      "logps/chosen": -34.200557708740234,
+      "logps/rejected": -34.543209075927734,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.15907494723796844,
+      "rewards/margins": 0.1246359720826149,
+      "rewards/rejected": 0.03443896025419235,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.941239356994629,
-      "logits/rejected": -1.9457323551177979,
-      "logps/chosen": -32.385948181152344,
-      "logps/rejected": -32.34455108642578,
-      "loss": 0.6805,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.057031381875276566,
-      "rewards/margins": 0.046979233622550964,
-      "rewards/rejected": 0.010052147321403027,
+      "logits/chosen": -1.9453551769256592,
+      "logits/rejected": -1.9498746395111084,
+      "logps/chosen": -32.2825927734375,
+      "logps/rejected": -32.25676727294922,
+      "loss": 0.4858,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.12937995791435242,
+      "rewards/margins": 0.05788033455610275,
+      "rewards/rejected": 0.07149962335824966,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0390255451202393,
-      "logits/rejected": -2.0370407104492188,
-      "logps/chosen": -32.15161895751953,
-      "logps/rejected": -31.304489135742188,
-      "loss": 0.6577,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.06245182827115059,
-      "rewards/margins": 0.08924683928489685,
-      "rewards/rejected": -0.02679501473903656,
+      "logits/chosen": -2.04329776763916,
+      "logits/rejected": -2.0412919521331787,
+      "logps/chosen": -32.015525817871094,
+      "logps/rejected": -31.169103622436523,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.15771625936031342,
+      "rewards/margins": 0.08974156528711319,
+      "rewards/rejected": 0.06797470152378082,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.233787775039673,
-      "eval_logits/rejected": -2.228933572769165,
-      "eval_logps/chosen": -34.04001235961914,
-      "eval_logps/rejected": -37.535667419433594,
-      "eval_loss": 0.6948937773704529,
-      "eval_rewards/accuracies": 0.5315614938735962,
-      "eval_rewards/chosen": -0.0038188453763723373,
-      "eval_rewards/margins": 0.00951432902365923,
-      "eval_rewards/rejected": -0.013333176262676716,
-      "eval_runtime": 145.7707,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.238145351409912,
+      "eval_logits/rejected": -2.2333030700683594,
+      "eval_logps/chosen": -33.86069107055664,
+      "eval_logps/rejected": -37.356990814208984,
+      "eval_loss": 0.49784979224205017,
+      "eval_rewards/accuracies": 0.5340532064437866,
+      "eval_rewards/chosen": 0.12170296907424927,
+      "eval_rewards/margins": 0.009964452125132084,
+      "eval_rewards/rejected": 0.11173851788043976,
+      "eval_runtime": 145.9039,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.994390845298767,
-      "logits/rejected": -1.991999864578247,
-      "logps/chosen": -33.1099739074707,
-      "logps/rejected": -34.02802276611328,
-      "loss": 0.6747,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.09369214624166489,
-      "rewards/margins": 0.0927465409040451,
-      "rewards/rejected": 0.0009456165134906769,
+      "logits/chosen": -1.9996016025543213,
+      "logits/rejected": -1.9972642660140991,
+      "logps/chosen": -32.95330810546875,
+      "logps/rejected": -33.88172912597656,
+      "loss": 0.4742,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.20335713028907776,
+      "rewards/margins": 0.10000836849212646,
+      "rewards/rejected": 0.10334876924753189,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0057625770568848,
-      "logits/rejected": -1.9974448680877686,
-      "logps/chosen": -32.31781768798828,
-      "logps/rejected": -32.132328033447266,
-      "loss": 0.6694,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0889604315161705,
-      "rewards/margins": 0.06853620707988739,
-      "rewards/rejected": 0.02042422816157341,
+      "logits/chosen": -2.0096044540405273,
+      "logits/rejected": -2.0012660026550293,
+      "logps/chosen": -32.153419494628906,
+      "logps/rejected": -31.961782455444336,
+      "loss": 0.4848,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.20403961837291718,
+      "rewards/margins": 0.06423385441303253,
+      "rewards/rejected": 0.13980577886104584,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0336880683898926,
-      "logits/rejected": -2.0257279872894287,
-      "logps/chosen": -30.32443618774414,
-      "logps/rejected": -32.07221221923828,
-      "loss": 0.6522,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.10589826107025146,
-      "rewards/margins": 0.11934350430965424,
-      "rewards/rejected": -0.013445250689983368,
+      "logits/chosen": -2.038412570953369,
+      "logits/rejected": -2.0304617881774902,
+      "logps/chosen": -30.165945053100586,
+      "logps/rejected": -31.880346298217773,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.21683910489082336,
+      "rewards/margins": 0.0959777683019638,
+      "rewards/rejected": 0.12086131423711777,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9642670154571533,
-      "logits/rejected": -1.9745124578475952,
-      "logps/chosen": -31.212024688720703,
-      "logps/rejected": -32.565834045410156,
-      "loss": 0.6272,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14534947276115417,
-      "rewards/margins": 0.16701875627040863,
-      "rewards/rejected": -0.02166926860809326,
+      "logits/chosen": -1.9683482646942139,
+      "logits/rejected": -1.9784959554672241,
+      "logps/chosen": -31.041671752929688,
+      "logps/rejected": -32.388893127441406,
+      "loss": 0.4621,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.264592707157135,
+      "rewards/margins": 0.16240297257900238,
+      "rewards/rejected": 0.10218973457813263,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8759937286376953,
-      "logits/rejected": -1.8771553039550781,
-      "logps/chosen": -33.917335510253906,
-      "logps/rejected": -34.8185920715332,
-      "loss": 0.6064,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.19274887442588806,
-      "rewards/margins": 0.235683411359787,
-      "rewards/rejected": -0.042934536933898926,
+      "logits/chosen": -1.8799880743026733,
+      "logits/rejected": -1.8811382055282593,
+      "logps/chosen": -33.686256408691406,
+      "logps/rejected": -34.564491271972656,
+      "loss": 0.4503,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.3545047640800476,
+      "rewards/margins": 0.21957096457481384,
+      "rewards/rejected": 0.134933739900589,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.927369475364685,
-      "logits/rejected": -1.9239553213119507,
-      "logps/chosen": -35.98323440551758,
-      "logps/rejected": -32.705108642578125,
-      "loss": 0.6464,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.13190819323062897,
-      "rewards/margins": 0.11955627053976059,
-      "rewards/rejected": 0.012351910583674908,
+      "logits/chosen": -1.931627869606018,
+      "logits/rejected": -1.9282808303833008,
+      "logps/chosen": -35.77287292480469,
+      "logps/rejected": -32.49742889404297,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.27916350960731506,
+      "rewards/margins": 0.12143567949533463,
+      "rewards/rejected": 0.15772780776023865,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.028454542160034,
-      "logits/rejected": -2.0210976600646973,
-      "logps/chosen": -33.50457763671875,
-      "logps/rejected": -31.386072158813477,
-      "loss": 0.6077,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.20529839396476746,
-      "rewards/margins": 0.22925393283367157,
-      "rewards/rejected": -0.02395555004477501,
+      "logits/chosen": -2.0332443714141846,
+      "logits/rejected": -2.0259475708007812,
+      "logps/chosen": -33.251670837402344,
+      "logps/rejected": -31.17826271057129,
+      "loss": 0.439,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.382335364818573,
+      "rewards/margins": 0.2608257234096527,
+      "rewards/rejected": 0.1215096265077591,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0345230102539062,
-      "logits/rejected": -2.0397608280181885,
-      "logps/chosen": -32.19758224487305,
-      "logps/rejected": -32.435523986816406,
-      "loss": 0.6083,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.24304144084453583,
-      "rewards/margins": 0.2083090990781784,
-      "rewards/rejected": 0.03473237156867981,
+      "logits/chosen": -2.040196657180786,
+      "logits/rejected": -2.045389413833618,
+      "logps/chosen": -31.95781898498535,
+      "logps/rejected": -32.18687057495117,
+      "loss": 0.4516,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4108788073062897,
+      "rewards/margins": 0.20208871364593506,
+      "rewards/rejected": 0.20879006385803223,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.035979747772217,
-      "logits/rejected": -2.0332181453704834,
-      "logps/chosen": -31.255151748657227,
-      "logps/rejected": -31.321630477905273,
-      "loss": 0.6326,
+      "logits/chosen": -2.040775775909424,
+      "logits/rejected": -2.038043737411499,
+      "logps/chosen": -31.056228637695312,
+      "logps/rejected": -31.10147476196289,
+      "loss": 0.4647,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.163661926984787,
-      "rewards/margins": 0.16605310142040253,
-      "rewards/rejected": -0.0023911758325994015,
+      "rewards/chosen": 0.30290743708610535,
+      "rewards/margins": 0.1511872559785843,
+      "rewards/rejected": 0.15172019600868225,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9058892726898193,
-      "logits/rejected": -1.910540223121643,
-      "logps/chosen": -31.33197021484375,
-      "logps/rejected": -32.817604064941406,
-      "loss": 0.6156,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.19201864302158356,
-      "rewards/margins": 0.21068540215492249,
-      "rewards/rejected": -0.01866675540804863,
+      "logits/chosen": -1.9112863540649414,
+      "logits/rejected": -1.9159523248672485,
+      "logps/chosen": -31.08163070678711,
+      "logps/rejected": -32.598487854003906,
+      "loss": 0.4453,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.3672560155391693,
+      "rewards/margins": 0.2325470894575119,
+      "rewards/rejected": 0.1347089260816574,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2316529750823975,
-      "eval_logits/rejected": -2.2268118858337402,
-      "eval_logps/chosen": -34.053470611572266,
-      "eval_logps/rejected": -37.557804107666016,
-      "eval_loss": 0.6943473219871521,
-      "eval_rewards/accuracies": 0.5191029906272888,
-      "eval_rewards/chosen": -0.013246187008917332,
-      "eval_rewards/margins": 0.015584951266646385,
-      "eval_rewards/rejected": -0.028831137344241142,
-      "eval_runtime": 145.8114,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.234708309173584,
+      "eval_logits/rejected": -2.229902505874634,
+      "eval_logps/chosen": -33.76850891113281,
+      "eval_logps/rejected": -37.29365921020508,
+      "eval_loss": 0.49275800585746765,
+      "eval_rewards/accuracies": 0.5369601845741272,
+      "eval_rewards/chosen": 0.18622951209545135,
+      "eval_rewards/margins": 0.030155813321471214,
+      "eval_rewards/rejected": 0.1560736745595932,
+      "eval_runtime": 145.5287,
+      "eval_samples_per_second": 2.357,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.01808762550354,
-      "logits/rejected": -2.0287296772003174,
-      "logps/chosen": -31.763586044311523,
-      "logps/rejected": -33.926063537597656,
-      "loss": 0.622,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.14540112018585205,
-      "rewards/margins": 0.1897541582584381,
-      "rewards/rejected": -0.04435301572084427,
+      "logits/chosen": -2.022939443588257,
+      "logits/rejected": -2.033546209335327,
+      "logps/chosen": -31.5457706451416,
+      "logps/rejected": -33.67595672607422,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.2978750169277191,
+      "rewards/margins": 0.16715146601200104,
+      "rewards/rejected": 0.13072356581687927,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.910658597946167,
-      "logits/rejected": -1.9254140853881836,
-      "logps/chosen": -29.832544326782227,
-      "logps/rejected": -31.612533569335938,
-      "loss": 0.6024,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.19529443979263306,
-      "rewards/margins": 0.23334410786628723,
-      "rewards/rejected": -0.038049641996622086,
+      "logits/chosen": -1.9158331155776978,
+      "logits/rejected": -1.9305419921875,
+      "logps/chosen": -29.57406997680664,
+      "logps/rejected": -31.42013168334961,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.37622833251953125,
+      "rewards/margins": 0.2795966863632202,
+      "rewards/rejected": 0.09663165360689163,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9673402309417725,
-      "logits/rejected": -1.9713077545166016,
-      "logps/chosen": -33.062660217285156,
-      "logps/rejected": -31.608203887939453,
-      "loss": 0.5911,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.24836687743663788,
-      "rewards/margins": 0.2895973324775696,
-      "rewards/rejected": -0.0412304513156414,
+      "logits/chosen": -1.9733517169952393,
+      "logits/rejected": -1.9773433208465576,
+      "logps/chosen": -32.83649444580078,
+      "logps/rejected": -31.40326499938965,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.40668267011642456,
+      "rewards/margins": 0.304456502199173,
+      "rewards/rejected": 0.10222617536783218,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9657704830169678,
-      "logits/rejected": -1.9439115524291992,
-      "logps/chosen": -33.82358932495117,
-      "logps/rejected": -35.09131622314453,
-      "loss": 0.5733,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.23111140727996826,
-      "rewards/margins": 0.32947883009910583,
-      "rewards/rejected": -0.09836738556623459,
+      "logits/chosen": -1.9713729619979858,
+      "logits/rejected": -1.9496545791625977,
+      "logps/chosen": -33.5602912902832,
+      "logps/rejected": -34.85026550292969,
+      "loss": 0.4227,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.41541916131973267,
+      "rewards/margins": 0.3450462222099304,
+      "rewards/rejected": 0.07037289440631866,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.007312536239624,
-      "logits/rejected": -2.003988742828369,
-      "logps/chosen": -32.68491744995117,
-      "logps/rejected": -36.24928283691406,
-      "loss": 0.6228,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.1681055724620819,
-      "rewards/margins": 0.18515849113464355,
-      "rewards/rejected": -0.01705293543636799,
+      "logits/chosen": -2.012058734893799,
+      "logits/rejected": -2.0087602138519287,
+      "logps/chosen": -32.46921157836914,
+      "logps/rejected": -36.012245178222656,
+      "loss": 0.4613,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.31910061836242676,
+      "rewards/margins": 0.17022350430488586,
+      "rewards/rejected": 0.1488770991563797,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8734614849090576,
-      "logits/rejected": -1.8710591793060303,
-      "logps/chosen": -33.96207809448242,
-      "logps/rejected": -35.5266227722168,
-      "loss": 0.6286,
+      "logits/chosen": -1.878584861755371,
+      "logits/rejected": -1.876153588294983,
+      "logps/chosen": -33.71690368652344,
+      "logps/rejected": -35.26210403442383,
+      "loss": 0.4631,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.15842413902282715,
-      "rewards/margins": 0.17382602393627167,
-      "rewards/rejected": -0.015401872806251049,
+      "rewards/chosen": 0.33004888892173767,
+      "rewards/margins": 0.1602855622768402,
+      "rewards/rejected": 0.16976332664489746,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8584483861923218,
-      "logits/rejected": -1.8560174703598022,
-      "logps/chosen": -34.1807975769043,
-      "logps/rejected": -31.80449867248535,
-      "loss": 0.6325,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.15176042914390564,
-      "rewards/margins": 0.17497751116752625,
-      "rewards/rejected": -0.02321707457304001,
+      "logits/chosen": -1.863705039024353,
+      "logits/rejected": -1.8611875772476196,
+      "logps/chosen": -33.94118881225586,
+      "logps/rejected": -31.579448699951172,
+      "loss": 0.4582,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.3194878101348877,
+      "rewards/margins": 0.18517015874385834,
+      "rewards/rejected": 0.13431766629219055,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9630708694458008,
-      "logits/rejected": -1.9525701999664307,
-      "logps/chosen": -35.00975036621094,
-      "logps/rejected": -31.84867286682129,
-      "loss": 0.5978,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2428235560655594,
-      "rewards/margins": 0.24820086359977722,
-      "rewards/rejected": -0.005377279128879309,
+      "logits/chosen": -1.96773362159729,
+      "logits/rejected": -1.957313895225525,
+      "logps/chosen": -34.72475814819336,
+      "logps/rejected": -31.634963989257812,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.4423200190067291,
+      "rewards/margins": 0.298100084066391,
+      "rewards/rejected": 0.14421990513801575,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0586793422698975,
-      "logits/rejected": -2.043766498565674,
-      "logps/chosen": -30.7253360748291,
-      "logps/rejected": -32.66551971435547,
-      "loss": 0.6462,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.13915367424488068,
-      "rewards/margins": 0.14873233437538147,
-      "rewards/rejected": -0.009578653611242771,
+      "logits/chosen": -2.0632526874542236,
+      "logits/rejected": -2.0484511852264404,
+      "logps/chosen": -30.412399291992188,
+      "logps/rejected": -32.339237213134766,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.3582096993923187,
+      "rewards/margins": 0.13939058780670166,
+      "rewards/rejected": 0.21881911158561707,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9287078380584717,
-      "logits/rejected": -1.92616868019104,
-      "logps/chosen": -32.39979553222656,
-      "logps/rejected": -30.87893295288086,
-      "loss": 0.5468,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.36911940574645996,
-      "rewards/margins": 0.4126061797142029,
-      "rewards/rejected": -0.04348675161600113,
+      "logits/chosen": -1.9345197677612305,
+      "logits/rejected": -1.9320701360702515,
+      "logps/chosen": -32.11776351928711,
+      "logps/rejected": -30.692516326904297,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.5665432810783386,
+      "rewards/margins": 0.4795381426811218,
+      "rewards/rejected": 0.08700509369373322,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2277145385742188,
-      "eval_logits/rejected": -2.222882032394409,
-      "eval_logps/chosen": -34.06844711303711,
-      "eval_logps/rejected": -37.586029052734375,
-      "eval_loss": 0.6902604699134827,
-      "eval_rewards/accuracies": 0.5191029906272888,
-      "eval_rewards/chosen": -0.023726314306259155,
-      "eval_rewards/margins": 0.024859989061951637,
-      "eval_rewards/rejected": -0.04858630895614624,
-      "eval_runtime": 145.7514,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.2321932315826416,
+      "eval_logits/rejected": -2.227381944656372,
+      "eval_logps/chosen": -33.75507736206055,
+      "eval_logps/rejected": -37.28935623168945,
+      "eval_loss": 0.49099090695381165,
+      "eval_rewards/accuracies": 0.5564784407615662,
+      "eval_rewards/chosen": 0.1956319361925125,
+      "eval_rewards/margins": 0.03654861077666283,
+      "eval_rewards/rejected": 0.15908333659172058,
+      "eval_runtime": 145.678,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 10.6875,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -2.0614776611328125,
-      "logits/rejected": -2.048677921295166,
-      "logps/chosen": -32.06366729736328,
-      "logps/rejected": -32.84763717651367,
-      "loss": 0.506,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.32530978322029114,
-      "rewards/margins": 0.46991363167762756,
-      "rewards/rejected": -0.14460383355617523,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.918630838394165,
+      "logits/rejected": -1.9154895544052124,
+      "logps/chosen": -31.033864974975586,
+      "logps/rejected": -33.57470703125,
+      "loss": 0.4357,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3986373543739319,
+      "rewards/margins": 0.28142648935317993,
+      "rewards/rejected": 0.11721093952655792,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 10.3125,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9717050790786743,
-      "logits/rejected": -1.9630792140960693,
-      "logps/chosen": -31.734317779541016,
-      "logps/rejected": -30.413864135742188,
-      "loss": 0.4921,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.4344688355922699,
-      "rewards/margins": 0.555200457572937,
-      "rewards/rejected": -0.12073162943124771,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.9683278799057007,
+      "logits/rejected": -1.9561859369277954,
+      "logps/chosen": -34.02568817138672,
+      "logps/rejected": -33.45148849487305,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.39926695823669434,
+      "rewards/margins": 0.3381231129169464,
+      "rewards/rejected": 0.06114383786916733,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 11.6875,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.908839464187622,
-      "logits/rejected": -1.9209339618682861,
-      "logps/chosen": -29.812292098999023,
-      "logps/rejected": -33.6970329284668,
-      "loss": 0.4445,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.46688413619995117,
-      "rewards/margins": 0.6732310056686401,
-      "rewards/rejected": -0.20634695887565613,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -2.0041966438293457,
+      "logits/rejected": -2.0028293132781982,
+      "logps/chosen": -32.882896423339844,
+      "logps/rejected": -32.24889373779297,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.4220260679721832,
+      "rewards/margins": 0.26054397225379944,
+      "rewards/rejected": 0.16148212552070618,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 15.5,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -1.871185064315796,
-      "logits/rejected": -1.8619896173477173,
-      "logps/chosen": -34.0085334777832,
-      "logps/rejected": -36.21852493286133,
-      "loss": 0.4426,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.48327580094337463,
-      "rewards/margins": 0.7365350127220154,
-      "rewards/rejected": -0.25325921177864075,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.0908923149108887,
+      "logits/rejected": -2.0752511024475098,
+      "logps/chosen": -33.49097442626953,
+      "logps/rejected": -32.81463623046875,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.4818909764289856,
+      "rewards/margins": 0.26951199769973755,
+      "rewards/rejected": 0.21237893402576447,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 10.75,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.9235925674438477,
-      "logits/rejected": -1.9242353439331055,
-      "logps/chosen": -33.64180374145508,
-      "logps/rejected": -34.13081359863281,
-      "loss": 0.4532,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.5344331860542297,
-      "rewards/margins": 0.7317206859588623,
-      "rewards/rejected": -0.1972874402999878,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9633783102035522,
+      "logits/rejected": -1.9625694751739502,
+      "logps/chosen": -32.55985641479492,
+      "logps/rejected": -32.24340057373047,
+      "loss": 0.4211,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.5055657029151917,
+      "rewards/margins": 0.3503406047821045,
+      "rewards/rejected": 0.15522508323192596,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 11.875,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -2.048530101776123,
-      "logits/rejected": -2.054663896560669,
-      "logps/chosen": -31.074010848999023,
-      "logps/rejected": -32.97708511352539,
-      "loss": 0.4932,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.34946271777153015,
-      "rewards/margins": 0.5656248331069946,
-      "rewards/rejected": -0.21616211533546448,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9187549352645874,
+      "logits/rejected": -1.9290361404418945,
+      "logps/chosen": -31.61210060119629,
+      "logps/rejected": -35.0199089050293,
+      "loss": 0.4489,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.41481319069862366,
+      "rewards/margins": 0.22661848366260529,
+      "rewards/rejected": 0.18819470703601837,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 21.125,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -1.9776489734649658,
-      "logits/rejected": -1.977195382118225,
-      "logps/chosen": -32.816444396972656,
-      "logps/rejected": -36.346214294433594,
-      "loss": 0.4709,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.4399483799934387,
-      "rewards/margins": 0.6474584937095642,
-      "rewards/rejected": -0.2075101137161255,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.0583438873291016,
+      "logits/rejected": -2.051846742630005,
+      "logps/chosen": -33.05150604248047,
+      "logps/rejected": -28.9710750579834,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.42032259702682495,
+      "rewards/margins": 0.2415778934955597,
+      "rewards/rejected": 0.17874471843242645,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 9.5,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -2.0090365409851074,
-      "logits/rejected": -2.004446506500244,
-      "logps/chosen": -30.068859100341797,
-      "logps/rejected": -29.55948829650879,
-      "loss": 0.4523,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.4492177367210388,
-      "rewards/margins": 0.6911332011222839,
-      "rewards/rejected": -0.24191546440124512,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9180545806884766,
+      "logits/rejected": -1.92022705078125,
+      "logps/chosen": -33.62676239013672,
+      "logps/rejected": -30.719894409179688,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4286006987094879,
+      "rewards/margins": 0.3216874301433563,
+      "rewards/rejected": 0.10691330581903458,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "grad_norm": 12.125,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.7796638011932373,
-      "logits/rejected": -1.786041498184204,
-      "logps/chosen": -31.63083267211914,
-      "logps/rejected": -36.7193603515625,
-      "loss": 0.3988,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.5284172296524048,
-      "rewards/margins": 0.9491678476333618,
-      "rewards/rejected": -0.4207506775856018,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "grad_norm": 9.875,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.9318805932998657,
-      "logits/rejected": -1.9255841970443726,
-      "logps/chosen": -33.00324249267578,
-      "logps/rejected": -32.683555603027344,
-      "loss": 0.4243,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.5806173086166382,
-      "rewards/margins": 0.8428903818130493,
-      "rewards/rejected": -0.26227301359176636,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.1981148719787598,
-      "eval_logits/rejected": -2.19327449798584,
-      "eval_logps/chosen": -34.17210388183594,
-      "eval_logps/rejected": -37.72929382324219,
-      "eval_loss": 0.6885781288146973,
-      "eval_rewards/accuracies": 0.5539867281913757,
-      "eval_rewards/chosen": -0.0962882786989212,
-      "eval_rewards/margins": 0.052584208548069,
-      "eval_rewards/rejected": -0.1488724797964096,
-      "eval_runtime": 145.4574,
-      "eval_samples_per_second": 2.358,
-      "eval_steps_per_second": 0.296,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "grad_norm": 10.875,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -1.9296133518218994,
-      "logits/rejected": -1.9369118213653564,
-      "logps/chosen": -28.550052642822266,
-      "logps/rejected": -29.74081802368164,
-      "loss": 0.403,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.3992263674736023,
-      "rewards/margins": 0.8490440249443054,
-      "rewards/rejected": -0.4498176574707031,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "grad_norm": 9.8125,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.947829246520996,
-      "logits/rejected": -1.946836233139038,
-      "logps/chosen": -33.56596755981445,
-      "logps/rejected": -31.192113876342773,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4793195128440857,
-      "rewards/margins": 0.6536680459976196,
-      "rewards/rejected": -0.17434856295585632,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "grad_norm": 13.0625,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.9415998458862305,
-      "logits/rejected": -1.9243113994598389,
-      "logps/chosen": -30.399789810180664,
-      "logps/rejected": -33.614166259765625,
-      "loss": 0.4155,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.40419188141822815,
-      "rewards/margins": 0.885710597038269,
-      "rewards/rejected": -0.4815187454223633,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "grad_norm": 12.0625,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -1.972794771194458,
-      "logits/rejected": -1.9749000072479248,
-      "logps/chosen": -35.72783660888672,
-      "logps/rejected": -35.48394775390625,
-      "loss": 0.3567,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.6191657185554504,
-      "rewards/margins": 1.0559546947479248,
-      "rewards/rejected": -0.43678897619247437,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "grad_norm": 10.3125,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -2.047752857208252,
-      "logits/rejected": -2.0474939346313477,
-      "logps/chosen": -31.872684478759766,
-      "logps/rejected": -33.99079895019531,
-      "loss": 0.423,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.5324660539627075,
-      "rewards/margins": 0.8339457511901855,
-      "rewards/rejected": -0.3014797270298004,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "grad_norm": 11.1875,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -1.9906551837921143,
-      "logits/rejected": -1.9831295013427734,
-      "logps/chosen": -31.529306411743164,
-      "logps/rejected": -36.79833984375,
-      "loss": 0.3453,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6136585474014282,
-      "rewards/margins": 1.1346721649169922,
-      "rewards/rejected": -0.5210134983062744,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "grad_norm": 14.25,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.8940445184707642,
-      "logits/rejected": -1.8905950784683228,
-      "logps/chosen": -33.06376266479492,
-      "logps/rejected": -33.512367248535156,
-      "loss": 0.3856,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.6079725623130798,
-      "rewards/margins": 1.0199116468429565,
-      "rewards/rejected": -0.4119391441345215,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 11.5625,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -2.0184760093688965,
-      "logits/rejected": -2.011504650115967,
-      "logps/chosen": -30.26607322692871,
-      "logps/rejected": -32.60536575317383,
-      "loss": 0.4501,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.393722802400589,
-      "rewards/margins": 0.8220372200012207,
-      "rewards/rejected": -0.42831435799598694,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "grad_norm": 8.8125,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9625924825668335,
-      "logits/rejected": -1.9641788005828857,
-      "logps/chosen": -34.393211364746094,
-      "logps/rejected": -34.28179168701172,
-      "loss": 0.3543,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.671985387802124,
-      "rewards/margins": 1.1201808452606201,
-      "rewards/rejected": -0.4481953978538513,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 16.5,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.8526058197021484,
-      "logits/rejected": -1.8502172231674194,
-      "logps/chosen": -32.287147521972656,
-      "logps/rejected": -37.31818771362305,
-      "loss": 0.353,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.619600236415863,
-      "rewards/margins": 1.2173068523406982,
-      "rewards/rejected": -0.5977065563201904,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.1931960582733154,
-      "eval_logits/rejected": -2.1883604526519775,
-      "eval_logps/chosen": -34.31940460205078,
-      "eval_logps/rejected": -37.908485412597656,
-      "eval_loss": 0.6900750398635864,
-      "eval_rewards/accuracies": 0.5568937063217163,
-      "eval_rewards/chosen": -0.199394091963768,
-      "eval_rewards/margins": 0.07491057366132736,
-      "eval_rewards/rejected": -0.27430465817451477,
-      "eval_runtime": 145.2894,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "grad_norm": 10.6875,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.9544782638549805,
-      "logits/rejected": -1.9592231512069702,
-      "logps/chosen": -32.51142883300781,
-      "logps/rejected": -32.14912414550781,
-      "loss": 0.3835,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.6621495485305786,
-      "rewards/margins": 1.0142595767974854,
-      "rewards/rejected": -0.3521100878715515,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "grad_norm": 11.25,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -1.983420968055725,
-      "logits/rejected": -1.994410753250122,
-      "logps/chosen": -30.491313934326172,
-      "logps/rejected": -32.20922088623047,
-      "loss": 0.3741,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.6293708682060242,
-      "rewards/margins": 1.120445728302002,
-      "rewards/rejected": -0.491074800491333,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "grad_norm": 5.9375,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.9113273620605469,
-      "logits/rejected": -1.9080307483673096,
-      "logps/chosen": -31.719308853149414,
-      "logps/rejected": -33.058982849121094,
-      "loss": 0.3945,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.49084219336509705,
-      "rewards/margins": 0.9781869053840637,
-      "rewards/rejected": -0.4873446822166443,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 10.0,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.7767927646636963,
-      "logits/rejected": -1.7859961986541748,
-      "logps/chosen": -31.72823715209961,
-      "logps/rejected": -32.537315368652344,
-      "loss": 0.391,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.5888075828552246,
-      "rewards/margins": 1.0657360553741455,
-      "rewards/rejected": -0.47692856192588806,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "grad_norm": 11.125,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.8712069988250732,
-      "logits/rejected": -1.8649717569351196,
-      "logps/chosen": -32.81023025512695,
-      "logps/rejected": -31.534648895263672,
-      "loss": 0.4096,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.6286527514457703,
-      "rewards/margins": 1.008962869644165,
-      "rewards/rejected": -0.38031005859375,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "grad_norm": 8.4375,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -1.9593925476074219,
-      "logits/rejected": -1.9573974609375,
-      "logps/chosen": -33.5708122253418,
-      "logps/rejected": -32.04071807861328,
-      "loss": 0.3942,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.700269877910614,
-      "rewards/margins": 1.0806026458740234,
-      "rewards/rejected": -0.380332887172699,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "grad_norm": 10.625,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.7931559085845947,
-      "logits/rejected": -1.7910410165786743,
-      "logps/chosen": -31.88705825805664,
-      "logps/rejected": -31.514333724975586,
-      "loss": 0.4211,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.6365472674369812,
-      "rewards/margins": 0.9949695467948914,
-      "rewards/rejected": -0.35842233896255493,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "grad_norm": 13.3125,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -1.9281076192855835,
-      "logits/rejected": -1.92474365234375,
-      "logps/chosen": -30.16890525817871,
-      "logps/rejected": -35.45796203613281,
-      "loss": 0.3493,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.6550183296203613,
-      "rewards/margins": 1.237276315689087,
-      "rewards/rejected": -0.582257866859436,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "grad_norm": 8.25,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.865663766860962,
-      "logits/rejected": -1.8691942691802979,
-      "logps/chosen": -28.939590454101562,
-      "logps/rejected": -31.212047576904297,
-      "loss": 0.4162,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.47280949354171753,
-      "rewards/margins": 0.86003577709198,
-      "rewards/rejected": -0.38722628355026245,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "grad_norm": 12.3125,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.902017593383789,
-      "logits/rejected": -1.9019527435302734,
-      "logps/chosen": -31.125457763671875,
-      "logps/rejected": -31.8260555267334,
-      "loss": 0.3554,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.7672521471977234,
-      "rewards/margins": 1.2103806734085083,
-      "rewards/rejected": -0.44312843680381775,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.174912452697754,
-      "eval_logits/rejected": -2.170095920562744,
-      "eval_logps/chosen": -34.24431228637695,
-      "eval_logps/rejected": -37.88401794433594,
-      "eval_loss": 0.6762823462486267,
-      "eval_rewards/accuracies": 0.5805647969245911,
-      "eval_rewards/chosen": -0.14683447778224945,
-      "eval_rewards/margins": 0.11034313589334488,
-      "eval_rewards/rejected": -0.2571776211261749,
-      "eval_runtime": 145.2157,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "grad_norm": 11.9375,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -1.9643291234970093,
-      "logits/rejected": -1.9650930166244507,
-      "logps/chosen": -33.864723205566406,
-      "logps/rejected": -33.84077072143555,
-      "loss": 0.3368,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.8157946467399597,
-      "rewards/margins": 1.3454043865203857,
-      "rewards/rejected": -0.5296097993850708,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "grad_norm": 8.9375,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -1.9274613857269287,
-      "logits/rejected": -1.9249248504638672,
-      "logps/chosen": -32.089725494384766,
-      "logps/rejected": -32.76671600341797,
-      "loss": 0.3571,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.7249698638916016,
-      "rewards/margins": 1.2127305269241333,
-      "rewards/rejected": -0.4877607822418213,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "grad_norm": 22.75,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.9384180307388306,
-      "logits/rejected": -1.9293851852416992,
-      "logps/chosen": -31.670080184936523,
-      "logps/rejected": -31.639263153076172,
-      "loss": 0.3787,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.7742272019386292,
-      "rewards/margins": 1.2669562101364136,
-      "rewards/rejected": -0.4927290380001068,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "grad_norm": 15.5625,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.9396206140518188,
-      "logits/rejected": -1.9363600015640259,
-      "logps/chosen": -31.236923217773438,
-      "logps/rejected": -33.08216857910156,
-      "loss": 0.3585,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6730504631996155,
-      "rewards/margins": 1.1369760036468506,
-      "rewards/rejected": -0.4639255106449127,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "grad_norm": 9.5625,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -1.9486452341079712,
-      "logits/rejected": -1.9509565830230713,
-      "logps/chosen": -32.633331298828125,
-      "logps/rejected": -34.409324645996094,
-      "loss": 0.3199,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.7142627835273743,
-      "rewards/margins": 1.2224125862121582,
-      "rewards/rejected": -0.5081498026847839,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "grad_norm": 15.875,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.8591312170028687,
-      "logits/rejected": -1.8595669269561768,
-      "logps/chosen": -32.87069320678711,
-      "logps/rejected": -32.203189849853516,
-      "loss": 0.3567,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.753296971321106,
-      "rewards/margins": 1.1704500913619995,
-      "rewards/rejected": -0.41715312004089355,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "grad_norm": 13.625,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.8990064859390259,
-      "logits/rejected": -1.8893133401870728,
-      "logps/chosen": -32.891273498535156,
-      "logps/rejected": -33.074562072753906,
-      "loss": 0.3066,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.7770041227340698,
-      "rewards/margins": 1.31374192237854,
-      "rewards/rejected": -0.5367377400398254,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "grad_norm": 9.0625,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.7967946529388428,
-      "logits/rejected": -1.7935699224472046,
-      "logps/chosen": -30.264007568359375,
-      "logps/rejected": -35.02149963378906,
-      "loss": 0.3142,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.8378661870956421,
-      "rewards/margins": 1.329910159111023,
-      "rewards/rejected": -0.4920438826084137,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "grad_norm": 7.8125,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.85686457157135,
-      "logits/rejected": -1.8561124801635742,
-      "logps/chosen": -33.96955108642578,
-      "logps/rejected": -36.205108642578125,
-      "loss": 0.2775,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9105154275894165,
-      "rewards/margins": 1.506414771080017,
-      "rewards/rejected": -0.5958994626998901,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "grad_norm": 6.90625,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.831987738609314,
-      "logits/rejected": -1.8347162008285522,
-      "logps/chosen": -31.40140724182129,
-      "logps/rejected": -36.432777404785156,
-      "loss": 0.3333,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.841008186340332,
-      "rewards/margins": 1.394784688949585,
-      "rewards/rejected": -0.5537765622138977,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.1596450805664062,
-      "eval_logits/rejected": -2.1548590660095215,
-      "eval_logps/chosen": -34.294071197509766,
-      "eval_logps/rejected": -37.937469482421875,
-      "eval_loss": 0.6812623739242554,
-      "eval_rewards/accuracies": 0.574335515499115,
-      "eval_rewards/chosen": -0.18166212737560272,
-      "eval_rewards/margins": 0.11293323338031769,
-      "eval_rewards/rejected": -0.2945953607559204,
-      "eval_runtime": 145.3544,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "grad_norm": 12.75,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -1.9378350973129272,
-      "logits/rejected": -1.9405092000961304,
-      "logps/chosen": -33.763160705566406,
-      "logps/rejected": -34.92665100097656,
-      "loss": 0.3812,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.7927051782608032,
-      "rewards/margins": 1.252252221107483,
-      "rewards/rejected": -0.4595469832420349,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "grad_norm": 11.625,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.7911384105682373,
-      "logits/rejected": -1.7852294445037842,
-      "logps/chosen": -33.655067443847656,
-      "logps/rejected": -33.061012268066406,
-      "loss": 0.3416,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.8247106671333313,
-      "rewards/margins": 1.3211290836334229,
-      "rewards/rejected": -0.49641841650009155,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 7.40625,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -1.9650695323944092,
-      "logits/rejected": -1.9620001316070557,
-      "logps/chosen": -29.55094337463379,
-      "logps/rejected": -32.38628005981445,
-      "loss": 0.302,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.7714294195175171,
-      "rewards/margins": 1.428086519241333,
-      "rewards/rejected": -0.6566572785377502,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "grad_norm": 8.4375,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.8031257390975952,
-      "logits/rejected": -1.8032245635986328,
-      "logps/chosen": -32.12765121459961,
-      "logps/rejected": -38.37054443359375,
-      "loss": 0.3055,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9348654747009277,
-      "rewards/margins": 1.4547784328460693,
-      "rewards/rejected": -0.5199130773544312,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "grad_norm": 6.0625,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.6882374286651611,
-      "logits/rejected": -1.6935392618179321,
-      "logps/chosen": -35.69883346557617,
-      "logps/rejected": -34.775413513183594,
-      "loss": 0.3555,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.8635395765304565,
-      "rewards/margins": 1.4256962537765503,
-      "rewards/rejected": -0.5621567964553833,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "grad_norm": 7.9375,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.8880552053451538,
-      "logits/rejected": -1.889639139175415,
-      "logps/chosen": -31.452478408813477,
-      "logps/rejected": -33.45410919189453,
-      "loss": 0.3277,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.8072429895401001,
-      "rewards/margins": 1.279667615890503,
-      "rewards/rejected": -0.4724246561527252,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 12.4375,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.8625637292861938,
-      "logits/rejected": -1.8615506887435913,
-      "logps/chosen": -30.956247329711914,
-      "logps/rejected": -35.569908142089844,
-      "loss": 0.3448,
-      "rewards/accuracies": 0.92083340883255,
-      "rewards/chosen": 0.8535542488098145,
-      "rewards/margins": 1.2327837944030762,
-      "rewards/rejected": -0.3792295455932617,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "grad_norm": 6.28125,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.8628917932510376,
-      "logits/rejected": -1.8622173070907593,
-      "logps/chosen": -33.29192352294922,
-      "logps/rejected": -32.4744758605957,
-      "loss": 0.2348,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.9495759010314941,
-      "rewards/margins": 1.6880419254302979,
-      "rewards/rejected": -0.7384659051895142,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "grad_norm": 7.5,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.8925044536590576,
-      "logits/rejected": -1.8910400867462158,
-      "logps/chosen": -28.264022827148438,
-      "logps/rejected": -33.81805419921875,
-      "loss": 0.2096,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.9094095230102539,
-      "rewards/margins": 1.7366279363632202,
-      "rewards/rejected": -0.8272183537483215,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "grad_norm": 4.46875,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.9434454441070557,
-      "logits/rejected": -1.9400180578231812,
-      "logps/chosen": -30.975006103515625,
-      "logps/rejected": -35.40107345581055,
-      "loss": 0.2025,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.016541838645935,
-      "rewards/margins": 1.8579689264297485,
-      "rewards/rejected": -0.8414268493652344,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.1460890769958496,
-      "eval_logits/rejected": -2.1413440704345703,
-      "eval_logps/chosen": -34.36545181274414,
-      "eval_logps/rejected": -38.040489196777344,
-      "eval_loss": 0.6800107359886169,
-      "eval_rewards/accuracies": 0.5660299062728882,
-      "eval_rewards/chosen": -0.23162823915481567,
-      "eval_rewards/margins": 0.13507823646068573,
-      "eval_rewards/rejected": -0.3667064905166626,
-      "eval_runtime": 145.2997,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "grad_norm": 6.8125,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.7730064392089844,
-      "logits/rejected": -1.765316367149353,
-      "logps/chosen": -32.71251678466797,
-      "logps/rejected": -35.768611907958984,
-      "loss": 0.1913,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.2206047773361206,
-      "rewards/margins": 2.066551446914673,
-      "rewards/rejected": -0.8459464907646179,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "grad_norm": 8.125,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.8485921621322632,
-      "logits/rejected": -1.8590046167373657,
-      "logps/chosen": -34.08565902709961,
-      "logps/rejected": -33.43722915649414,
-      "loss": 0.2332,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.1270233392715454,
-      "rewards/margins": 1.8310768604278564,
-      "rewards/rejected": -0.7040534615516663,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "grad_norm": 10.375,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.897798776626587,
-      "logits/rejected": -1.9024550914764404,
-      "logps/chosen": -33.517730712890625,
-      "logps/rejected": -30.249008178710938,
-      "loss": 0.2494,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.9695348739624023,
-      "rewards/margins": 1.6560699939727783,
-      "rewards/rejected": -0.6865350008010864,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "grad_norm": 6.15625,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.8966840505599976,
-      "logits/rejected": -1.9048032760620117,
-      "logps/chosen": -33.26880645751953,
-      "logps/rejected": -31.018238067626953,
-      "loss": 0.2145,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.063457727432251,
-      "rewards/margins": 1.8828351497650146,
-      "rewards/rejected": -0.8193773031234741,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "grad_norm": 9.5,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.826229453086853,
-      "logits/rejected": -1.8167400360107422,
-      "logps/chosen": -30.364776611328125,
-      "logps/rejected": -33.12417984008789,
-      "loss": 0.227,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.9998935461044312,
-      "rewards/margins": 1.777157187461853,
-      "rewards/rejected": -0.7772639393806458,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "grad_norm": 5.5,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -1.9638983011245728,
-      "logits/rejected": -1.954012155532837,
-      "logps/chosen": -24.777915954589844,
-      "logps/rejected": -31.053104400634766,
-      "loss": 0.226,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.012310266494751,
-      "rewards/margins": 1.8388469219207764,
-      "rewards/rejected": -0.8265368342399597,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "grad_norm": 6.0625,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.8167579174041748,
-      "logits/rejected": -1.8181308507919312,
-      "logps/chosen": -32.69607925415039,
-      "logps/rejected": -31.112274169921875,
-      "loss": 0.2008,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.1458479166030884,
-      "rewards/margins": 1.9366874694824219,
-      "rewards/rejected": -0.7908396124839783,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "grad_norm": 6.8125,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.8225730657577515,
-      "logits/rejected": -1.823504090309143,
-      "logps/chosen": -31.072418212890625,
-      "logps/rejected": -34.2416877746582,
-      "loss": 0.263,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.9217823147773743,
-      "rewards/margins": 1.7553189992904663,
-      "rewards/rejected": -0.8335369229316711,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "grad_norm": 6.5625,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.7258434295654297,
-      "logits/rejected": -1.7242809534072876,
-      "logps/chosen": -34.12605667114258,
-      "logps/rejected": -37.38144302368164,
-      "loss": 0.2016,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.077201247215271,
-      "rewards/margins": 2.130145311355591,
-      "rewards/rejected": -1.0529439449310303,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "grad_norm": 6.1875,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.8766018152236938,
-      "logits/rejected": -1.8697988986968994,
-      "logps/chosen": -30.475255966186523,
-      "logps/rejected": -34.02910232543945,
-      "loss": 0.2153,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.069166898727417,
-      "rewards/margins": 1.9588909149169922,
-      "rewards/rejected": -0.8897239565849304,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.1339919567108154,
-      "eval_logits/rejected": -2.129246950149536,
-      "eval_logps/chosen": -34.38914108276367,
-      "eval_logps/rejected": -38.06317138671875,
-      "eval_loss": 0.6866059899330139,
-      "eval_rewards/accuracies": 0.5834717750549316,
-      "eval_rewards/chosen": -0.2482113242149353,
-      "eval_rewards/margins": 0.13437683880329132,
-      "eval_rewards/rejected": -0.38258814811706543,
-      "eval_runtime": 145.4001,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "grad_norm": 10.0,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.913761854171753,
-      "logits/rejected": -1.9091001749038696,
-      "logps/chosen": -32.0454216003418,
-      "logps/rejected": -35.850093841552734,
-      "loss": 0.2031,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0585777759552002,
-      "rewards/margins": 1.9836070537567139,
-      "rewards/rejected": -0.9250291585922241,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "grad_norm": 6.8125,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.832114815711975,
-      "logits/rejected": -1.8371769189834595,
-      "logps/chosen": -33.69664764404297,
-      "logps/rejected": -33.70267868041992,
-      "loss": 0.2011,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.156354308128357,
-      "rewards/margins": 1.9982963800430298,
-      "rewards/rejected": -0.8419421315193176,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "grad_norm": 6.4375,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -1.9312747716903687,
-      "logits/rejected": -1.9220285415649414,
-      "logps/chosen": -30.794708251953125,
-      "logps/rejected": -35.72539520263672,
-      "loss": 0.192,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.0431582927703857,
-      "rewards/margins": 1.9409774541854858,
-      "rewards/rejected": -0.8978190422058105,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "grad_norm": 7.9375,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.8571876287460327,
-      "logits/rejected": -1.8539724349975586,
-      "logps/chosen": -30.002208709716797,
-      "logps/rejected": -33.57603454589844,
-      "loss": 0.2511,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.9003726840019226,
-      "rewards/margins": 1.6778360605239868,
-      "rewards/rejected": -0.7774633765220642,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "grad_norm": 4.3125,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.7950204610824585,
-      "logits/rejected": -1.8024400472640991,
-      "logps/chosen": -31.523014068603516,
-      "logps/rejected": -36.79169464111328,
-      "loss": 0.2492,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.9436267614364624,
-      "rewards/margins": 1.812860131263733,
-      "rewards/rejected": -0.8692331314086914,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "grad_norm": 4.53125,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.8054288625717163,
-      "logits/rejected": -1.798541784286499,
-      "logps/chosen": -32.300254821777344,
-      "logps/rejected": -38.06441116333008,
-      "loss": 0.1874,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0697839260101318,
-      "rewards/margins": 2.095707893371582,
-      "rewards/rejected": -1.0259240865707397,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "grad_norm": 6.0,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.8083324432373047,
-      "logits/rejected": -1.8088661432266235,
-      "logps/chosen": -29.700159072875977,
-      "logps/rejected": -34.11577606201172,
-      "loss": 0.225,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0753717422485352,
-      "rewards/margins": 1.8695356845855713,
-      "rewards/rejected": -0.7941640615463257,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "grad_norm": 7.53125,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.8105262517929077,
-      "logits/rejected": -1.821751594543457,
-      "logps/chosen": -31.371307373046875,
-      "logps/rejected": -35.32282638549805,
-      "loss": 0.2071,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.1937681436538696,
-      "rewards/margins": 2.0278971195220947,
-      "rewards/rejected": -0.8341289758682251,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "grad_norm": 6.1875,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.8809820413589478,
-      "logits/rejected": -1.8785299062728882,
-      "logps/chosen": -34.623592376708984,
-      "logps/rejected": -33.443870544433594,
-      "loss": 0.2779,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.0482189655303955,
-      "rewards/margins": 1.779466986656189,
-      "rewards/rejected": -0.7312478423118591,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "grad_norm": 8.3125,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.8533103466033936,
-      "logits/rejected": -1.8574621677398682,
-      "logps/chosen": -32.96641540527344,
-      "logps/rejected": -35.521705627441406,
-      "loss": 0.2381,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0478132963180542,
-      "rewards/margins": 1.7727762460708618,
-      "rewards/rejected": -0.7249631285667419,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.125450372695923,
-      "eval_logits/rejected": -2.1207265853881836,
-      "eval_logps/chosen": -34.40937423706055,
-      "eval_logps/rejected": -38.111175537109375,
-      "eval_loss": 0.6821461319923401,
-      "eval_rewards/accuracies": 0.5863787531852722,
-      "eval_rewards/chosen": -0.2623753547668457,
-      "eval_rewards/margins": 0.15381217002868652,
-      "eval_rewards/rejected": -0.41618749499320984,
-      "eval_runtime": 145.3424,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "grad_norm": 17.25,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.8625215291976929,
-      "logits/rejected": -1.8674328327178955,
-      "logps/chosen": -31.423959732055664,
-      "logps/rejected": -33.53556442260742,
-      "loss": 0.2722,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.9635237455368042,
-      "rewards/margins": 1.6735379695892334,
-      "rewards/rejected": -0.7100142240524292,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "grad_norm": 12.6875,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.8214099407196045,
-      "logits/rejected": -1.8238738775253296,
-      "logps/chosen": -28.987659454345703,
-      "logps/rejected": -32.98705291748047,
-      "loss": 0.2111,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0205867290496826,
-      "rewards/margins": 1.922104835510254,
-      "rewards/rejected": -0.9015182256698608,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "grad_norm": 4.59375,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -1.8745782375335693,
-      "logits/rejected": -1.869202971458435,
-      "logps/chosen": -32.560997009277344,
-      "logps/rejected": -34.77830505371094,
-      "loss": 0.2251,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.9996814727783203,
-      "rewards/margins": 1.881895661354065,
-      "rewards/rejected": -0.8822140693664551,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "grad_norm": 14.75,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.8110053539276123,
-      "logits/rejected": -1.815009355545044,
-      "logps/chosen": -34.393035888671875,
-      "logps/rejected": -35.58574676513672,
-      "loss": 0.2344,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.9524880647659302,
-      "rewards/margins": 1.7688013315200806,
-      "rewards/rejected": -0.8163133859634399,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "grad_norm": 7.34375,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.8082377910614014,
-      "logits/rejected": -1.821096420288086,
-      "logps/chosen": -30.48015785217285,
-      "logps/rejected": -34.21984100341797,
-      "loss": 0.2333,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.9679109454154968,
-      "rewards/margins": 1.8077707290649414,
-      "rewards/rejected": -0.8398596048355103,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "grad_norm": 15.1875,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.796105980873108,
-      "logits/rejected": -1.789453148841858,
-      "logps/chosen": -30.575389862060547,
-      "logps/rejected": -34.41447830200195,
-      "loss": 0.21,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.1087596416473389,
-      "rewards/margins": 1.8763185739517212,
-      "rewards/rejected": -0.7675586938858032,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "grad_norm": 6.1875,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.9221141338348389,
-      "logits/rejected": -1.9228368997573853,
-      "logps/chosen": -32.064552307128906,
-      "logps/rejected": -34.02466583251953,
-      "loss": 0.2118,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0377662181854248,
-      "rewards/margins": 1.8637428283691406,
-      "rewards/rejected": -0.8259769678115845,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "grad_norm": 9.0,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.8529789447784424,
-      "logits/rejected": -1.8518245220184326,
-      "logps/chosen": -32.69489288330078,
-      "logps/rejected": -34.210750579833984,
-      "loss": 0.2684,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.9764569997787476,
-      "rewards/margins": 1.6896979808807373,
-      "rewards/rejected": -0.7132409811019897,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "grad_norm": 5.53125,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -1.8765695095062256,
-      "logits/rejected": -1.8690354824066162,
-      "logps/chosen": -32.69994354248047,
-      "logps/rejected": -32.084373474121094,
-      "loss": 0.2211,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0681509971618652,
-      "rewards/margins": 1.7981399297714233,
-      "rewards/rejected": -0.7299890518188477,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "grad_norm": 5.0625,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.8488590717315674,
-      "logits/rejected": -1.8478235006332397,
-      "logps/chosen": -33.83041763305664,
-      "logps/rejected": -37.73679733276367,
-      "loss": 0.1898,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1115968227386475,
-      "rewards/margins": 2.0804603099823,
-      "rewards/rejected": -0.9688636660575867,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.123544454574585,
-      "eval_logits/rejected": -2.118812084197998,
-      "eval_logps/chosen": -34.41644287109375,
-      "eval_logps/rejected": -38.11098098754883,
-      "eval_loss": 0.6857725381851196,
-      "eval_rewards/accuracies": 0.5830564498901367,
-      "eval_rewards/chosen": -0.26732438802719116,
-      "eval_rewards/margins": 0.14872850477695465,
-      "eval_rewards/rejected": -0.4160528779029846,
-      "eval_runtime": 145.3786,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "grad_norm": 6.53125,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -1.8964908123016357,
-      "logits/rejected": -1.9019794464111328,
-      "logps/chosen": -34.118003845214844,
-      "logps/rejected": -36.824615478515625,
-      "loss": 0.2423,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9443825483322144,
-      "rewards/margins": 1.8411028385162354,
-      "rewards/rejected": -0.8967200517654419,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "grad_norm": 4.71875,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.8656179904937744,
-      "logits/rejected": -1.86355721950531,
-      "logps/chosen": -31.237468719482422,
-      "logps/rejected": -34.223793029785156,
-      "loss": 0.2295,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.9651158452033997,
-      "rewards/margins": 1.8338441848754883,
-      "rewards/rejected": -0.8687284588813782,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "grad_norm": 5.375,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.7683290243148804,
-      "logits/rejected": -1.7710100412368774,
-      "logps/chosen": -28.29164695739746,
-      "logps/rejected": -31.67474937438965,
-      "loss": 0.1929,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0198917388916016,
-      "rewards/margins": 2.0206894874572754,
-      "rewards/rejected": -1.0007976293563843,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "grad_norm": 5.625,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -1.8653761148452759,
-      "logits/rejected": -1.8621852397918701,
-      "logps/chosen": -31.63905906677246,
-      "logps/rejected": -33.515663146972656,
-      "loss": 0.2347,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.095284342765808,
-      "rewards/margins": 1.7874319553375244,
-      "rewards/rejected": -0.6921476125717163,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "grad_norm": 8.5625,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.7846425771713257,
-      "logits/rejected": -1.7783504724502563,
-      "logps/chosen": -26.89322853088379,
-      "logps/rejected": -33.4926643371582,
-      "loss": 0.2239,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.8713330030441284,
-      "rewards/margins": 1.8123865127563477,
-      "rewards/rejected": -0.9410533905029297,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "grad_norm": 5.375,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.8979135751724243,
-      "logits/rejected": -1.8953993320465088,
-      "logps/chosen": -31.6197452545166,
-      "logps/rejected": -36.48524856567383,
-      "loss": 0.2063,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0334270000457764,
-      "rewards/margins": 2.009028196334839,
-      "rewards/rejected": -0.9756014943122864,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "grad_norm": 4.9375,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -1.88443124294281,
-      "logits/rejected": -1.8823401927947998,
-      "logps/chosen": -30.127737045288086,
-      "logps/rejected": -33.89713668823242,
-      "loss": 0.172,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0406690835952759,
-      "rewards/margins": 2.1138501167297363,
-      "rewards/rejected": -1.0731807947158813,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "grad_norm": 5.25,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.8772131204605103,
-      "logits/rejected": -1.865001916885376,
-      "logps/chosen": -34.52227783203125,
-      "logps/rejected": -35.57610321044922,
-      "loss": 0.1854,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0661174058914185,
-      "rewards/margins": 2.119356870651245,
-      "rewards/rejected": -1.0532395839691162,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "grad_norm": 6.375,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.7719604969024658,
-      "logits/rejected": -1.7666189670562744,
-      "logps/chosen": -33.58086395263672,
-      "logps/rejected": -35.4622802734375,
-      "loss": 0.1812,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.2381782531738281,
-      "rewards/margins": 2.0952985286712646,
-      "rewards/rejected": -0.857120156288147,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "grad_norm": 5.90625,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.8176746368408203,
-      "logits/rejected": -1.8208776712417603,
-      "logps/chosen": -31.216379165649414,
-      "logps/rejected": -32.722049713134766,
-      "loss": 0.2231,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.9791001081466675,
-      "rewards/margins": 1.8442093133926392,
-      "rewards/rejected": -0.8651090860366821,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.122251272201538,
-      "eval_logits/rejected": -2.11751651763916,
-      "eval_logps/chosen": -34.40974426269531,
-      "eval_logps/rejected": -38.12571716308594,
-      "eval_loss": 0.6779673099517822,
-      "eval_rewards/accuracies": 0.5888704061508179,
-      "eval_rewards/chosen": -0.26263338327407837,
-      "eval_rewards/margins": 0.1637314409017563,
-      "eval_rewards/rejected": -0.42636486887931824,
-      "eval_runtime": 145.3842,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "grad_norm": 10.1875,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.8395841121673584,
-      "logits/rejected": -1.8350778818130493,
-      "logps/chosen": -29.8420352935791,
-      "logps/rejected": -32.26592254638672,
-      "loss": 0.1948,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0021460056304932,
-      "rewards/margins": 1.952625036239624,
-      "rewards/rejected": -0.9504791498184204,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "grad_norm": 5.90625,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.8544647693634033,
-      "logits/rejected": -1.8489763736724854,
-      "logps/chosen": -33.709232330322266,
-      "logps/rejected": -36.026329040527344,
-      "loss": 0.2003,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0952959060668945,
-      "rewards/margins": 2.114811420440674,
-      "rewards/rejected": -1.0195155143737793,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "grad_norm": 4.46875,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.8562214374542236,
-      "logits/rejected": -1.8429136276245117,
-      "logps/chosen": -31.71542739868164,
-      "logps/rejected": -35.530879974365234,
-      "loss": 0.2252,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.9650102853775024,
-      "rewards/margins": 1.9527349472045898,
-      "rewards/rejected": -0.9877246618270874,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "grad_norm": 4.34375,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.902470350265503,
-      "logits/rejected": -1.8999955654144287,
-      "logps/chosen": -33.41004180908203,
-      "logps/rejected": -32.994361877441406,
-      "loss": 0.198,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.1360009908676147,
-      "rewards/margins": 1.9441492557525635,
-      "rewards/rejected": -0.8081479072570801,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "grad_norm": 5.3125,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.8268474340438843,
-      "logits/rejected": -1.823965072631836,
-      "logps/chosen": -32.80628967285156,
-      "logps/rejected": -36.46531677246094,
-      "loss": 0.1702,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1129032373428345,
-      "rewards/margins": 2.2806363105773926,
-      "rewards/rejected": -1.1677329540252686,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "grad_norm": 5.25,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.8951988220214844,
-      "logits/rejected": -1.8899462223052979,
-      "logps/chosen": -29.183536529541016,
-      "logps/rejected": -33.69340133666992,
-      "loss": 0.1818,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1040847301483154,
-      "rewards/margins": 2.0216822624206543,
-      "rewards/rejected": -0.9175974130630493,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "grad_norm": 4.625,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.9220434427261353,
-      "logits/rejected": -1.926945447921753,
-      "logps/chosen": -31.057159423828125,
-      "logps/rejected": -32.91054153442383,
-      "loss": 0.184,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.157687783241272,
-      "rewards/margins": 2.01811146736145,
-      "rewards/rejected": -0.8604235649108887,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "grad_norm": 7.5,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.8148295879364014,
-      "logits/rejected": -1.8222448825836182,
-      "logps/chosen": -33.42390823364258,
-      "logps/rejected": -33.99805450439453,
-      "loss": 0.2006,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0383344888687134,
-      "rewards/margins": 2.018185615539551,
-      "rewards/rejected": -0.9798509478569031,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "grad_norm": 4.84375,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.7262365818023682,
-      "logits/rejected": -1.728548288345337,
-      "logps/chosen": -31.478628158569336,
-      "logps/rejected": -37.968849182128906,
-      "loss": 0.1982,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1572868824005127,
-      "rewards/margins": 2.147489547729492,
-      "rewards/rejected": -0.9902026057243347,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "grad_norm": 5.03125,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.7753759622573853,
-      "logits/rejected": -1.7788490056991577,
-      "logps/chosen": -31.414981842041016,
-      "logps/rejected": -33.79729461669922,
-      "loss": 0.164,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2611814737319946,
-      "rewards/margins": 2.1943717002868652,
-      "rewards/rejected": -0.9331903457641602,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.1220953464508057,
-      "eval_logits/rejected": -2.117370843887329,
-      "eval_logps/chosen": -34.417152404785156,
-      "eval_logps/rejected": -38.11579513549805,
-      "eval_loss": 0.6833942532539368,
-      "eval_rewards/accuracies": 0.594684362411499,
-      "eval_rewards/chosen": -0.2678225636482239,
-      "eval_rewards/margins": 0.15159977972507477,
-      "eval_rewards/rejected": -0.41942232847213745,
-      "eval_runtime": 145.412,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "grad_norm": 3.578125,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.88503098487854,
-      "logits/rejected": -1.864648461341858,
-      "logps/chosen": -29.50347328186035,
-      "logps/rejected": -36.774391174316406,
-      "loss": 0.1881,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.1269481182098389,
-      "rewards/margins": 2.2525367736816406,
-      "rewards/rejected": -1.1255886554718018,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "grad_norm": 5.90625,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -1.8611704111099243,
-      "logits/rejected": -1.866959571838379,
-      "logps/chosen": -31.242517471313477,
-      "logps/rejected": -33.74761199951172,
-      "loss": 0.1726,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.2538880109786987,
-      "rewards/margins": 2.133692979812622,
-      "rewards/rejected": -0.8798049688339233,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "grad_norm": 4.21875,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.8369661569595337,
-      "logits/rejected": -1.8319981098175049,
-      "logps/chosen": -31.567821502685547,
-      "logps/rejected": -36.19134521484375,
-      "loss": 0.1725,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0182814598083496,
-      "rewards/margins": 2.151616096496582,
-      "rewards/rejected": -1.1333343982696533,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "grad_norm": 4.9375,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.6985762119293213,
-      "logits/rejected": -1.708148717880249,
-      "logps/chosen": -32.04993438720703,
-      "logps/rejected": -32.1507453918457,
-      "loss": 0.1954,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2247600555419922,
-      "rewards/margins": 2.006398916244507,
-      "rewards/rejected": -0.7816389799118042,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "grad_norm": 5.78125,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.756463646888733,
-      "logits/rejected": -1.7611831426620483,
-      "logps/chosen": -32.128456115722656,
-      "logps/rejected": -34.41259002685547,
-      "loss": 0.1564,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.27573561668396,
-      "rewards/margins": 2.1211793422698975,
-      "rewards/rejected": -0.8454439043998718,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "grad_norm": 18.5,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.7851018905639648,
-      "logits/rejected": -1.7904930114746094,
-      "logps/chosen": -32.26166534423828,
-      "logps/rejected": -35.997657775878906,
-      "loss": 0.2004,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.098996877670288,
-      "rewards/margins": 2.1032118797302246,
-      "rewards/rejected": -1.0042150020599365,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "grad_norm": 3.8125,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -1.8785173892974854,
-      "logits/rejected": -1.8740142583847046,
-      "logps/chosen": -31.18601417541504,
-      "logps/rejected": -32.262332916259766,
-      "loss": 0.1895,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.014768362045288,
-      "rewards/margins": 1.9600728750228882,
-      "rewards/rejected": -0.9453045129776001,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "grad_norm": 6.28125,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.7922194004058838,
-      "logits/rejected": -1.7942945957183838,
-      "logps/chosen": -28.180294036865234,
-      "logps/rejected": -32.01648712158203,
-      "loss": 0.2023,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0353037118911743,
-      "rewards/margins": 1.9259191751480103,
-      "rewards/rejected": -0.8906152844429016,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "grad_norm": 7.4375,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.7903833389282227,
-      "logits/rejected": -1.7950242757797241,
-      "logps/chosen": -30.506296157836914,
-      "logps/rejected": -33.995018005371094,
-      "loss": 0.2087,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0592371225357056,
-      "rewards/margins": 1.7641029357910156,
-      "rewards/rejected": -0.7048658132553101,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "grad_norm": 4.0625,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.8554019927978516,
-      "logits/rejected": -1.8586347103118896,
-      "logps/chosen": -33.235443115234375,
-      "logps/rejected": -34.80227279663086,
-      "loss": 0.1562,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1865932941436768,
-      "rewards/margins": 2.1516919136047363,
-      "rewards/rejected": -0.9650983810424805,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.121932029724121,
-      "eval_logits/rejected": -2.117196798324585,
-      "eval_logps/chosen": -34.41535568237305,
-      "eval_logps/rejected": -38.13964080810547,
-      "eval_loss": 0.6752650141716003,
-      "eval_rewards/accuracies": 0.5921927094459534,
-      "eval_rewards/chosen": -0.2665621340274811,
-      "eval_rewards/margins": 0.16955220699310303,
-      "eval_rewards/rejected": -0.4361143708229065,
-      "eval_runtime": 145.4569,
-      "eval_samples_per_second": 2.358,
-      "eval_steps_per_second": 0.296,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "grad_norm": 6.46875,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -1.9231650829315186,
-      "logits/rejected": -1.9162681102752686,
-      "logps/chosen": -32.11163330078125,
-      "logps/rejected": -35.49402618408203,
-      "loss": 0.186,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1836400032043457,
-      "rewards/margins": 2.0582594871520996,
-      "rewards/rejected": -0.8746196627616882,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "grad_norm": 8.6875,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.8132683038711548,
-      "logits/rejected": -1.8242778778076172,
-      "logps/chosen": -32.496726989746094,
-      "logps/rejected": -34.891380310058594,
-      "loss": 0.175,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.156802773475647,
-      "rewards/margins": 2.067012071609497,
-      "rewards/rejected": -0.9102091789245605,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "grad_norm": 7.53125,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.8921226263046265,
-      "logits/rejected": -1.9039920568466187,
-      "logps/chosen": -33.318809509277344,
-      "logps/rejected": -35.047691345214844,
-      "loss": 0.1674,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.219396948814392,
-      "rewards/margins": 2.2059476375579834,
-      "rewards/rejected": -0.9865506887435913,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "grad_norm": 7.875,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.8827718496322632,
-      "logits/rejected": -1.8849990367889404,
-      "logps/chosen": -31.633159637451172,
-      "logps/rejected": -35.23392868041992,
-      "loss": 0.2007,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1026588678359985,
-      "rewards/margins": 2.0644314289093018,
-      "rewards/rejected": -0.9617727398872375,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "grad_norm": 5.1875,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.8658161163330078,
-      "logits/rejected": -1.8587331771850586,
-      "logps/chosen": -33.893577575683594,
-      "logps/rejected": -35.00482940673828,
-      "loss": 0.2008,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.1723241806030273,
-      "rewards/margins": 2.036421298980713,
-      "rewards/rejected": -0.864096999168396,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "grad_norm": 5.0625,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.833505630493164,
-      "logits/rejected": -1.8394966125488281,
-      "logps/chosen": -29.668813705444336,
-      "logps/rejected": -35.166725158691406,
-      "loss": 0.1749,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.0810542106628418,
-      "rewards/margins": 2.1287617683410645,
-      "rewards/rejected": -1.0477073192596436,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "grad_norm": 4.90625,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.7813646793365479,
-      "logits/rejected": -1.779421091079712,
-      "logps/chosen": -29.22164535522461,
-      "logps/rejected": -31.763259887695312,
-      "loss": 0.2002,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.136735439300537,
-      "rewards/margins": 1.9550540447235107,
-      "rewards/rejected": -0.8183186650276184,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "grad_norm": 6.53125,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -1.956352949142456,
-      "logits/rejected": -1.9507420063018799,
-      "logps/chosen": -33.838050842285156,
-      "logps/rejected": -33.383323669433594,
-      "loss": 0.1868,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.013047456741333,
-      "rewards/margins": 1.941677451133728,
-      "rewards/rejected": -0.928629994392395,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "grad_norm": 7.21875,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.8494259119033813,
-      "logits/rejected": -1.8517593145370483,
-      "logps/chosen": -27.075820922851562,
-      "logps/rejected": -29.148422241210938,
-      "loss": 0.2343,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.980392336845398,
-      "rewards/margins": 1.768689513206482,
-      "rewards/rejected": -0.788297176361084,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "grad_norm": 6.15625,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.8568594455718994,
-      "logits/rejected": -1.8512837886810303,
-      "logps/chosen": -31.454471588134766,
-      "logps/rejected": -33.362735748291016,
-      "loss": 0.2163,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.9424249529838562,
-      "rewards/margins": 1.918071985244751,
-      "rewards/rejected": -0.97564697265625,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.121995687484741,
-      "eval_logits/rejected": -2.1172635555267334,
-      "eval_logps/chosen": -34.41799545288086,
-      "eval_logps/rejected": -38.11918640136719,
-      "eval_loss": 0.6830941438674927,
-      "eval_rewards/accuracies": 0.5801494717597961,
-      "eval_rewards/chosen": -0.26841017603874207,
-      "eval_rewards/margins": 0.15338675677776337,
-      "eval_rewards/rejected": -0.42179691791534424,
-      "eval_runtime": 145.2579,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "grad_norm": 8.625,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.7834112644195557,
-      "logits/rejected": -1.7875878810882568,
-      "logps/chosen": -32.74296951293945,
-      "logps/rejected": -35.563575744628906,
-      "loss": 0.2238,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9996176958084106,
-      "rewards/margins": 1.8068501949310303,
-      "rewards/rejected": -0.807232677936554,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "grad_norm": 4.625,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.8380959033966064,
-      "logits/rejected": -1.8364956378936768,
-      "logps/chosen": -29.186996459960938,
-      "logps/rejected": -33.88913345336914,
-      "loss": 0.2166,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0830271244049072,
-      "rewards/margins": 1.9926173686981201,
-      "rewards/rejected": -0.9095903635025024,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "grad_norm": 8.3125,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -1.9377511739730835,
-      "logits/rejected": -1.9299259185791016,
-      "logps/chosen": -26.640117645263672,
-      "logps/rejected": -30.038944244384766,
-      "loss": 0.1959,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.9959781765937805,
-      "rewards/margins": 1.9531135559082031,
-      "rewards/rejected": -0.9571350812911987,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 6.03125,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.8405107259750366,
-      "logits/rejected": -1.8302587270736694,
-      "logps/chosen": -31.55389976501465,
-      "logps/rejected": -36.86316680908203,
-      "loss": 0.1727,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.058610200881958,
-      "rewards/margins": 2.0750041007995605,
-      "rewards/rejected": -1.0163936614990234,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.2216110908365869,
-      "train_runtime": 10768.4727,
-      "train_samples_per_second": 1.144,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.45851393303313814,
+      "train_runtime": 3252.0771,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,