diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.6944,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2413 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866639494895935,
-      "logits/rejected": -1.870951533317566,
-      "logps/chosen": -36.991004943847656,
-      "logps/rejected": -33.64551544189453,
-      "loss": 0.4968,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.009355764836072922,
-      "rewards/margins": 0.018620232120156288,
-      "rewards/rejected": -0.009264465421438217,
+      "logits/chosen": -1.8664441108703613,
+      "logits/rejected": -1.8707667589187622,
+      "logps/chosen": -36.99871826171875,
+      "logps/rejected": -33.66032028198242,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.0047301375307142735,
+      "rewards/margins": 0.022878725081682205,
+      "rewards/rejected": -0.018148591741919518,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9975944757461548,
-      "logits/rejected": -2.000239133834839,
-      "logps/chosen": -29.655420303344727,
-      "logps/rejected": -29.04937171936035,
-      "loss": 0.5034,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007936288602650166,
-      "rewards/margins": -0.016377601772546768,
-      "rewards/rejected": 0.008441311307251453,
+      "logits/chosen": -1.9972795248031616,
+      "logits/rejected": -1.999922513961792,
+      "logps/chosen": -29.64887046813965,
+      "logps/rejected": -29.054330825805664,
+      "loss": 0.756,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.0040070666000247,
+      "rewards/margins": -0.009474089369177818,
+      "rewards/rejected": 0.0054670218378305435,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.920345664024353,
-      "logits/rejected": -1.917649507522583,
-      "logps/chosen": -31.40158462524414,
-      "logps/rejected": -33.22117614746094,
-      "loss": 0.499,
+      "logits/chosen": -1.9204769134521484,
+      "logits/rejected": -1.9177768230438232,
+      "logps/chosen": -31.404077529907227,
+      "logps/rejected": -33.22195053100586,
+      "loss": 0.7112,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.008706120774149895,
-      "rewards/margins": 0.010335424914956093,
-      "rewards/rejected": -0.0016293060034513474,
+      "rewards/chosen": 0.007212291471660137,
+      "rewards/margins": 0.009304212406277657,
+      "rewards/rejected": -0.0020919169764965773,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.016953706741333,
-      "logits/rejected": -2.008241891860962,
-      "logps/chosen": -32.55894088745117,
-      "logps/rejected": -32.48887634277344,
-      "loss": 0.501,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.010718915611505508,
-      "rewards/margins": -0.0035588580649346113,
-      "rewards/rejected": 0.014277773909270763,
+      "logits/chosen": -2.0172927379608154,
+      "logits/rejected": -2.008545398712158,
+      "logps/chosen": -32.57653045654297,
+      "logps/rejected": -32.520172119140625,
+      "loss": 0.7334,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00016416943981312215,
+      "rewards/margins": 0.004665405489504337,
+      "rewards/rejected": -0.004501235671341419,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8621622323989868,
-      "logits/rejected": -1.851413369178772,
-      "logps/chosen": -33.5220832824707,
-      "logps/rejected": -35.4012565612793,
-      "loss": 0.5034,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.021240826696157455,
-      "rewards/margins": -0.009411826729774475,
-      "rewards/rejected": 0.03065265715122223,
+      "logits/chosen": -1.8620569705963135,
+      "logits/rejected": -1.8512712717056274,
+      "logps/chosen": -33.55234146118164,
+      "logps/rejected": -35.454689025878906,
+      "loss": 0.7458,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003082751529291272,
+      "rewards/margins": 0.004490534774959087,
+      "rewards/rejected": -0.0014077809173613787,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9412288665771484,
-      "logits/rejected": -1.9431712627410889,
-      "logps/chosen": -32.506317138671875,
-      "logps/rejected": -33.202842712402344,
-      "loss": 0.4846,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.05631808191537857,
-      "rewards/margins": 0.07519452273845673,
-      "rewards/rejected": -0.018876442685723305,
+      "logits/chosen": -1.940212607383728,
+      "logits/rejected": -1.9421584606170654,
+      "logps/chosen": -32.56269454956055,
+      "logps/rejected": -33.21684265136719,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.022492902353405952,
+      "rewards/margins": 0.04977109283208847,
+      "rewards/rejected": -0.027278194203972816,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0726475715637207,
-      "logits/rejected": -2.077623128890991,
-      "logps/chosen": -33.90425491333008,
-      "logps/rejected": -36.56460189819336,
-      "loss": 0.4903,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.05054063722491264,
-      "rewards/margins": 0.04797897860407829,
-      "rewards/rejected": 0.002561659086495638,
+      "logits/chosen": -2.0728747844696045,
+      "logits/rejected": -2.0778398513793945,
+      "logps/chosen": -33.98863983154297,
+      "logps/rejected": -36.62779235839844,
+      "loss": 0.7641,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -9.316801879322156e-05,
+      "rewards/margins": 0.035260457545518875,
+      "rewards/rejected": -0.035353630781173706,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9338359832763672,
-      "logits/rejected": -1.9369510412216187,
-      "logps/chosen": -34.21551513671875,
-      "logps/rejected": -34.527488708496094,
-      "loss": 0.4797,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1273767203092575,
-      "rewards/margins": 0.08842838555574417,
-      "rewards/rejected": 0.038948334753513336,
+      "logits/chosen": -1.935132384300232,
+      "logits/rejected": -1.9382661581039429,
+      "logps/chosen": -34.321903228759766,
+      "logps/rejected": -34.638282775878906,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.06354494392871857,
+      "rewards/margins": 0.09106875956058502,
+      "rewards/rejected": -0.027523810043931007,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9420888423919678,
-      "logits/rejected": -1.9465935230255127,
-      "logps/chosen": -32.2774772644043,
-      "logps/rejected": -32.245147705078125,
-      "loss": 0.4887,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.11396588385105133,
-      "rewards/margins": 0.04570900648832321,
-      "rewards/rejected": 0.06825686991214752,
+      "logits/chosen": -1.9444854259490967,
+      "logits/rejected": -1.9490203857421875,
+      "logps/chosen": -32.41173553466797,
+      "logps/rejected": -32.337852478027344,
+      "loss": 0.7499,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.033411428332328796,
+      "rewards/margins": 0.020780622959136963,
+      "rewards/rejected": 0.012630810961127281,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.040524482727051,
-      "logits/rejected": -2.0385289192199707,
-      "logps/chosen": -32.03118133544922,
-      "logps/rejected": -31.205692291259766,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.12579473853111267,
-      "rewards/margins": 0.08948288857936859,
-      "rewards/rejected": 0.036311857402324677,
+      "logits/chosen": -2.042165994644165,
+      "logits/rejected": -2.0401768684387207,
+      "logps/chosen": -32.13690185546875,
+      "logps/rejected": -31.2872314453125,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.06236031651496887,
+      "rewards/margins": 0.0749734491109848,
+      "rewards/rejected": -0.012613129802048206,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.234907865524292,
-      "eval_logits/rejected": -2.2300779819488525,
-      "eval_logps/chosen": -33.90293884277344,
-      "eval_logps/rejected": -37.394290924072266,
-      "eval_loss": 0.4987158477306366,
-      "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.0789671465754509,
-      "eval_rewards/margins": 0.005569713190197945,
-      "eval_rewards/rejected": 0.0733974277973175,
-      "eval_runtime": 145.9867,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.2366247177124023,
+      "eval_logits/rejected": -2.231769323348999,
+      "eval_logps/chosen": -34.01411819458008,
+      "eval_logps/rejected": -37.50873947143555,
+      "eval_loss": 0.7449910640716553,
+      "eval_rewards/accuracies": 0.5240863561630249,
+      "eval_rewards/chosen": 0.01226096786558628,
+      "eval_rewards/margins": 0.007533104624599218,
+      "eval_rewards/rejected": 0.0047278618440032005,
+      "eval_runtime": 145.783,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9966247081756592,
-      "logits/rejected": -1.9942585229873657,
-      "logps/chosen": -33.005943298339844,
-      "logps/rejected": -33.908599853515625,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.14272192120552063,
-      "rewards/margins": 0.07025818526744843,
-      "rewards/rejected": 0.07246376574039459,
+      "logits/chosen": -1.9979403018951416,
+      "logits/rejected": -1.9955459833145142,
+      "logps/chosen": -33.12257766723633,
+      "logps/rejected": -33.99717712402344,
+      "loss": 0.8547,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07274619489908218,
+      "rewards/margins": 0.05342693254351616,
+      "rewards/rejected": 0.019319258630275726,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0064611434936523,
-      "logits/rejected": -1.9981132745742798,
-      "logps/chosen": -32.16605758666992,
-      "logps/rejected": -31.968135833740234,
-      "loss": 0.4883,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.16730833053588867,
-      "rewards/margins": 0.0512862391769886,
-      "rewards/rejected": 0.11602209508419037,
+      "logits/chosen": -2.009084463119507,
+      "logits/rejected": -2.0007264614105225,
+      "logps/chosen": -32.33968734741211,
+      "logps/rejected": -32.13513946533203,
+      "loss": 0.7104,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06312941759824753,
+      "rewards/margins": 0.047309644520282745,
+      "rewards/rejected": 0.015819773077964783,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0353620052337646,
-      "logits/rejected": -2.0273985862731934,
-      "logps/chosen": -30.145355224609375,
-      "logps/rejected": -31.892078399658203,
-      "loss": 0.477,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.19821622967720032,
-      "rewards/margins": 0.10165951400995255,
-      "rewards/rejected": 0.09655670821666718,
+      "logits/chosen": -2.0365753173828125,
+      "logits/rejected": -2.028564453125,
+      "logps/chosen": -30.363454818725586,
+      "logps/rejected": -32.06770706176758,
+      "loss": 0.71,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06735612452030182,
+      "rewards/margins": 0.07617705315351486,
+      "rewards/rejected": -0.008820934221148491,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.965121865272522,
-      "logits/rejected": -1.9753497838974,
-      "logps/chosen": -31.059722900390625,
-      "logps/rejected": -32.391807556152344,
-      "loss": 0.4695,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.21596479415893555,
-      "rewards/margins": 0.13012099266052246,
-      "rewards/rejected": 0.08584379404783249,
+      "logits/chosen": -1.9657623767852783,
+      "logits/rejected": -1.976011872291565,
+      "logps/chosen": -31.213130950927734,
+      "logps/rejected": -32.592655181884766,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.12391922622919083,
+      "rewards/margins": 0.1585826873779297,
+      "rewards/rejected": -0.03466347977519035,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8772761821746826,
-      "logits/rejected": -1.8784434795379639,
-      "logps/chosen": -33.682029724121094,
-      "logps/rejected": -34.56599044799805,
-      "loss": 0.4566,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.30639737844467163,
-      "rewards/margins": 0.19163966178894043,
-      "rewards/rejected": 0.11475770175457001,
+      "logits/chosen": -1.8778575658798218,
+      "logits/rejected": -1.879020094871521,
+      "logps/chosen": -33.998287200927734,
+      "logps/rejected": -34.83546829223633,
+      "loss": 0.566,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.11664185672998428,
+      "rewards/margins": 0.1635705530643463,
+      "rewards/rejected": -0.04692870378494263,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9283088445663452,
-      "logits/rejected": -1.9249660968780518,
-      "logps/chosen": -35.78284454345703,
-      "logps/rejected": -32.46989059448242,
-      "loss": 0.4802,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.23330166935920715,
-      "rewards/margins": 0.08158595860004425,
-      "rewards/rejected": 0.1517156958580017,
+      "logits/chosen": -1.9302040338516235,
+      "logits/rejected": -1.9267680644989014,
+      "logps/chosen": -36.0306510925293,
+      "logps/rejected": -32.73199462890625,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.08461670577526093,
+      "rewards/margins": 0.09016285836696625,
+      "rewards/rejected": -0.005546141881495714,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0310397148132324,
-      "logits/rejected": -2.0237185955047607,
-      "logps/chosen": -33.21892547607422,
-      "logps/rejected": -31.1767520904541,
-      "loss": 0.443,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.347359299659729,
-      "rewards/margins": 0.24229975044727325,
-      "rewards/rejected": 0.10505956411361694,
+      "logits/chosen": -2.031188726425171,
+      "logits/rejected": -2.0238261222839355,
+      "logps/chosen": -33.55583953857422,
+      "logps/rejected": -31.381237030029297,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.14521491527557373,
+      "rewards/margins": 0.16284644603729248,
+      "rewards/rejected": -0.017631549388170242,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.037632703781128,
-      "logits/rejected": -2.0428080558776855,
-      "logps/chosen": -31.97260093688965,
-      "logps/rejected": -32.14643096923828,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.34331104159355164,
-      "rewards/margins": 0.1400861144065857,
-      "rewards/rejected": 0.20322492718696594,
+      "logits/chosen": -2.037135362625122,
+      "logits/rejected": -2.0424118041992188,
+      "logps/chosen": -32.33055114746094,
+      "logps/rejected": -32.500144958496094,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.12854143977165222,
+      "rewards/margins": 0.13754449784755707,
+      "rewards/rejected": -0.009003056213259697,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0373456478118896,
-      "logits/rejected": -2.0346312522888184,
-      "logps/chosen": -31.077922821044922,
-      "logps/rejected": -31.089244842529297,
-      "loss": 0.4725,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.24661760032176971,
-      "rewards/margins": 0.10923590511083603,
-      "rewards/rejected": 0.13738170266151428,
+      "logits/chosen": -2.0384888648986816,
+      "logits/rejected": -2.0357367992401123,
+      "logps/chosen": -31.326950073242188,
+      "logps/rejected": -31.338672637939453,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09720095247030258,
+      "rewards/margins": 0.10947537422180176,
+      "rewards/rejected": -0.012274429202079773,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9075931310653687,
-      "logits/rejected": -1.9122419357299805,
-      "logps/chosen": -31.088512420654297,
-      "logps/rejected": -32.591243743896484,
-      "loss": 0.4548,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.3106623888015747,
-      "rewards/margins": 0.190850168466568,
-      "rewards/rejected": 0.11981221288442612,
+      "logits/chosen": -1.908755898475647,
+      "logits/rejected": -1.9134235382080078,
+      "logps/chosen": -31.36275863647461,
+      "logps/rejected": -32.82594680786133,
+      "loss": 0.59,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.14611336588859558,
+      "rewards/margins": 0.16712167859077454,
+      "rewards/rejected": -0.02100830152630806,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.233458995819092,
-      "eval_logits/rejected": -2.2286527156829834,
-      "eval_logps/chosen": -33.769569396972656,
-      "eval_logps/rejected": -37.28458786010742,
-      "eval_loss": 0.4955619275569916,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.15899108350276947,
-      "eval_rewards/margins": 0.019772635772824287,
-      "eval_rewards/rejected": 0.13921843469142914,
-      "eval_runtime": 145.6335,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.233200788497925,
+      "eval_logits/rejected": -2.228351354598999,
+      "eval_logps/chosen": -34.069149017333984,
+      "eval_logps/rejected": -37.57521057128906,
+      "eval_loss": 0.750046968460083,
+      "eval_rewards/accuracies": 0.5365448594093323,
+      "eval_rewards/chosen": -0.020758595317602158,
+      "eval_rewards/margins": 0.014396170154213905,
+      "eval_rewards/rejected": -0.035154763609170914,
+      "eval_runtime": 145.6956,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.020599365234375,
-      "logits/rejected": -2.0312187671661377,
-      "logps/chosen": -31.51775550842285,
-      "logps/rejected": -33.68301773071289,
-      "loss": 0.462,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2721289098262787,
-      "rewards/margins": 0.16431835293769836,
-      "rewards/rejected": 0.10781057924032211,
+      "logits/chosen": -2.0207877159118652,
+      "logits/rejected": -2.0314865112304688,
+      "logps/chosen": -31.7946834564209,
+      "logps/rejected": -33.94416046142578,
+      "loss": 0.544,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.10597337782382965,
+      "rewards/margins": 0.1548517644405365,
+      "rewards/rejected": -0.04887838289141655,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9131078720092773,
-      "logits/rejected": -1.9278091192245483,
-      "logps/chosen": -29.563827514648438,
-      "logps/rejected": -31.399028778076172,
-      "loss": 0.4463,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.3286263942718506,
-      "rewards/margins": 0.23313641548156738,
-      "rewards/rejected": 0.09548995643854141,
+      "logits/chosen": -1.9139292240142822,
+      "logits/rejected": -1.9287306070327759,
+      "logps/chosen": -29.91892433166504,
+      "logps/rejected": -31.5737247467041,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11556863784790039,
+      "rewards/margins": 0.12489677965641022,
+      "rewards/rejected": -0.00932813435792923,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.969555139541626,
-      "logits/rejected": -1.9735543727874756,
-      "logps/chosen": -32.82552719116211,
-      "logps/rejected": -31.39508628845215,
-      "loss": 0.4396,
+      "logits/chosen": -1.9701712131500244,
+      "logits/rejected": -1.974172592163086,
+      "logps/chosen": -33.17791748046875,
+      "logps/rejected": -31.603836059570312,
+      "loss": 0.4933,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.3551639914512634,
-      "rewards/margins": 0.26263612508773804,
-      "rewards/rejected": 0.0925278514623642,
+      "rewards/chosen": 0.1437317281961441,
+      "rewards/margins": 0.1764504611492157,
+      "rewards/rejected": -0.03271873667836189,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9689340591430664,
-      "logits/rejected": -1.947165846824646,
-      "logps/chosen": -33.572689056396484,
-      "logps/rejected": -34.84184265136719,
-      "loss": 0.4355,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3486354947090149,
-      "rewards/margins": 0.2832663357257843,
-      "rewards/rejected": 0.0653691440820694,
+      "logits/chosen": -1.9691722393035889,
+      "logits/rejected": -1.9472358226776123,
+      "logps/chosen": -33.92795181274414,
+      "logps/rejected": -35.06544876098633,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.13547900319099426,
+      "rewards/margins": 0.20427203178405762,
+      "rewards/rejected": -0.06879300624132156,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0102758407592773,
-      "logits/rejected": -2.006990909576416,
-      "logps/chosen": -32.49989700317383,
-      "logps/rejected": -35.99126434326172,
-      "loss": 0.472,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.2551037669181824,
-      "rewards/margins": 0.11490776389837265,
-      "rewards/rejected": 0.1401960253715515,
+      "logits/chosen": -2.010617971420288,
+      "logits/rejected": -2.0072851181030273,
+      "logps/chosen": -32.785011291503906,
+      "logps/rejected": -36.283348083496094,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08403309434652328,
+      "rewards/margins": 0.11908762156963348,
+      "rewards/rejected": -0.035054538398981094,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.877345323562622,
-      "logits/rejected": -1.8749182224273682,
-      "logps/chosen": -33.7043571472168,
-      "logps/rejected": -35.259422302246094,
-      "loss": 0.4671,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.2904246747493744,
-      "rewards/margins": 0.14330394566059113,
-      "rewards/rejected": 0.14712072908878326,
+      "logits/chosen": -1.8787453174591064,
+      "logits/rejected": -1.8763244152069092,
+      "logps/chosen": -34.000633239746094,
+      "logps/rejected": -35.53338623046875,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11265683174133301,
+      "rewards/margins": 0.1299169510602951,
+      "rewards/rejected": -0.017260130494832993,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8626378774642944,
-      "logits/rejected": -1.8601328134536743,
-      "logps/chosen": -33.90363311767578,
-      "logps/rejected": -31.588598251342773,
-      "loss": 0.457,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.29638105630874634,
-      "rewards/margins": 0.1867421716451645,
-      "rewards/rejected": 0.10963888466358185,
+      "logits/chosen": -1.864057183265686,
+      "logits/rejected": -1.861581802368164,
+      "logps/chosen": -34.23812484741211,
+      "logps/rejected": -31.777530670166016,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.09568403661251068,
+      "rewards/margins": 0.09940369427204132,
+      "rewards/rejected": -0.003719669533893466,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.965528130531311,
-      "logits/rejected": -1.9551260471343994,
-      "logps/chosen": -34.74420166015625,
-      "logps/rejected": -31.622207641601562,
-      "loss": 0.4442,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3674633204936981,
-      "rewards/margins": 0.23619195818901062,
-      "rewards/rejected": 0.1312713772058487,
+      "logits/chosen": -1.9680726528167725,
+      "logits/rejected": -1.957493782043457,
+      "logps/chosen": -35.05353546142578,
+      "logps/rejected": -31.8691463470459,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18186399340629578,
+      "rewards/margins": 0.19875726103782654,
+      "rewards/rejected": -0.01689324900507927,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0614726543426514,
-      "logits/rejected": -2.0466229915618896,
-      "logps/chosen": -30.37356948852539,
-      "logps/rejected": -32.3351936340332,
-      "loss": 0.4671,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.33033472299575806,
-      "rewards/margins": 0.14034779369831085,
-      "rewards/rejected": 0.1899869292974472,
+      "logits/chosen": -2.0636587142944336,
+      "logits/rejected": -2.048659563064575,
+      "logps/chosen": -30.76055908203125,
+      "logps/rejected": -32.65589141845703,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09814020991325378,
+      "rewards/margins": 0.10057313740253448,
+      "rewards/rejected": -0.0024329207371920347,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9327377080917358,
-      "logits/rejected": -1.9302759170532227,
-      "logps/chosen": -32.0928955078125,
-      "logps/rejected": -30.652236938476562,
-      "loss": 0.41,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.5005279779434204,
-      "rewards/margins": 0.4017825126647949,
-      "rewards/rejected": 0.09874548017978668,
+      "logits/chosen": -1.9355440139770508,
+      "logits/rejected": -1.9329853057861328,
+      "logps/chosen": -32.57345199584961,
+      "logps/rejected": -30.86557388305664,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.21219508349895477,
+      "rewards/margins": 0.24145305156707764,
+      "rewards/rejected": -0.029257934540510178,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2308623790740967,
-      "eval_logits/rejected": -2.226059913635254,
-      "eval_logps/chosen": -33.76142501831055,
-      "eval_logps/rejected": -37.284828186035156,
-      "eval_loss": 0.4937135577201843,
-      "eval_rewards/accuracies": 0.5361295938491821,
-      "eval_rewards/chosen": 0.16387464106082916,
-      "eval_rewards/margins": 0.02479831501841545,
-      "eval_rewards/rejected": 0.1390763521194458,
-      "eval_runtime": 145.6766,
+      "eval_logits/chosen": -2.233344554901123,
+      "eval_logits/rejected": -2.2284841537475586,
+      "eval_logps/chosen": -34.06221008300781,
+      "eval_logps/rejected": -37.56574249267578,
+      "eval_loss": 0.7655012607574463,
+      "eval_rewards/accuracies": 0.5070598125457764,
+      "eval_rewards/chosen": -0.016592998057603836,
+      "eval_rewards/margins": 0.012880506925284863,
+      "eval_rewards/rejected": -0.029473504051566124,
+      "eval_runtime": 145.6769,
       "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -1.9166145324707031,
-      "logits/rejected": -1.9134905338287354,
-      "logps/chosen": -31.025157928466797,
-      "logps/rejected": -33.5601692199707,
-      "loss": 0.4452,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.34691327810287476,
-      "rewards/margins": 0.23772580921649933,
-      "rewards/rejected": 0.10918750613927841,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.9186370372772217,
+      "logits/rejected": -1.9153635501861572,
+      "logps/chosen": -31.363632202148438,
+      "logps/rejected": -33.790061950683594,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1438317447900772,
+      "rewards/margins": 0.17257718741893768,
+      "rewards/rejected": -0.02874544821679592,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9646613597869873,
-      "logits/rejected": -1.952566146850586,
-      "logps/chosen": -34.02196502685547,
-      "logps/rejected": -33.40004348754883,
-      "loss": 0.4395,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.34446340799331665,
-      "rewards/margins": 0.2611829340457916,
-      "rewards/rejected": 0.08328045904636383,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.9695978164672852,
+      "logits/rejected": -1.9573185443878174,
+      "logps/chosen": -34.42280960083008,
+      "logps/rejected": -33.63968276977539,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.103958860039711,
+      "rewards/margins": 0.1644643247127533,
+      "rewards/rejected": -0.06050548702478409,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.9993693828582764,
-      "logits/rejected": -1.9980556964874268,
-      "logps/chosen": -32.787845611572266,
-      "logps/rejected": -32.25823211669922,
-      "loss": 0.4341,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4187692701816559,
-      "rewards/margins": 0.285956472158432,
-      "rewards/rejected": 0.1328127533197403,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -2.0046210289001465,
+      "logits/rejected": -2.003176689147949,
+      "logps/chosen": -33.34724426269531,
+      "logps/rejected": -32.53419876098633,
+      "loss": 0.5611,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.08312927186489105,
+      "rewards/margins": 0.11589743942022324,
+      "rewards/rejected": -0.032768163830041885,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -2.0861244201660156,
-      "logits/rejected": -2.0705418586730957,
-      "logps/chosen": -33.38449478149414,
-      "logps/rejected": -32.79410171508789,
-      "loss": 0.4348,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.4769386351108551,
-      "rewards/margins": 0.28257787227630615,
-      "rewards/rejected": 0.19436076283454895,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.091379404067993,
+      "logits/rejected": -2.0756208896636963,
+      "logps/chosen": -33.90237808227539,
+      "logps/rejected": -33.127288818359375,
+      "loss": 0.4859,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.16620711982250214,
+      "rewards/margins": 0.17176242172718048,
+      "rewards/rejected": -0.005555284209549427,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.959221601486206,
-      "logits/rejected": -1.9584068059921265,
-      "logps/chosen": -32.52511215209961,
-      "logps/rejected": -32.22833251953125,
-      "loss": 0.4289,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.45419034361839294,
-      "rewards/margins": 0.31210166215896606,
-      "rewards/rejected": 0.14208868145942688,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9639079570770264,
+      "logits/rejected": -1.9629888534545898,
+      "logps/chosen": -32.960304260253906,
+      "logps/rejected": -32.5081787109375,
+      "loss": 0.5048,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.1930757313966751,
+      "rewards/margins": 0.21889439225196838,
+      "rewards/rejected": -0.02581869065761566,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -1.9131596088409424,
-      "logits/rejected": -1.9233753681182861,
-      "logps/chosen": -31.385852813720703,
-      "logps/rejected": -35.03350067138672,
-      "loss": 0.4234,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.49130287766456604,
-      "rewards/margins": 0.338148295879364,
-      "rewards/rejected": 0.15315461158752441,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9195839166641235,
+      "logits/rejected": -1.929896593093872,
+      "logps/chosen": -31.997264862060547,
+      "logps/rejected": -35.334999084472656,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12445466220378876,
+      "rewards/margins": 0.15219925343990326,
+      "rewards/rejected": -0.027744591236114502,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -2.0467209815979004,
-      "logits/rejected": -2.0402884483337402,
-      "logps/chosen": -32.84780502319336,
-      "logps/rejected": -28.970489501953125,
-      "loss": 0.4219,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.48249703645706177,
-      "rewards/margins": 0.32893475890159607,
-      "rewards/rejected": 0.15356223285198212,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.058486223220825,
+      "logits/rejected": -2.051971912384033,
+      "logps/chosen": -33.44487380981445,
+      "logps/rejected": -29.261219024658203,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12425780296325684,
+      "rewards/margins": 0.1451319307088852,
+      "rewards/rejected": -0.020874138921499252,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -1.905468225479126,
-      "logits/rejected": -1.9076448678970337,
-      "logps/chosen": -33.3749885559082,
-      "logps/rejected": -30.71591567993164,
-      "loss": 0.4091,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.5184367895126343,
-      "rewards/margins": 0.42440906167030334,
-      "rewards/rejected": 0.09402772784233093,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9182488918304443,
+      "logits/rejected": -1.9204342365264893,
+      "logps/chosen": -33.968048095703125,
+      "logps/rejected": -30.890918731689453,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16260387003421783,
+      "rewards/margins": 0.17357683181762695,
+      "rewards/rejected": -0.010972961783409119,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.9335657358169556,
-      "logits/rejected": -1.9322980642318726,
-      "logps/chosen": -33.33202362060547,
-      "logps/rejected": -35.62676239013672,
-      "loss": 0.3702,
-      "rewards/accuracies": 0.7583333253860474,
-      "rewards/chosen": 0.6316612362861633,
-      "rewards/margins": 0.6141165494918823,
-      "rewards/rejected": 0.017544690519571304,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.8687912225723267,
-      "logits/rejected": -1.860594391822815,
-      "logps/chosen": -30.623647689819336,
-      "logps/rejected": -36.03453826904297,
-      "loss": 0.3497,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.6901980042457581,
-      "rewards/margins": 0.7465595006942749,
-      "rewards/rejected": -0.05636153370141983,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.2161529064178467,
-      "eval_logits/rejected": -2.2113351821899414,
-      "eval_logps/chosen": -33.672698974609375,
-      "eval_logps/rejected": -37.206199645996094,
-      "eval_loss": 0.4927479326725006,
-      "eval_rewards/accuracies": 0.5651993155479431,
-      "eval_rewards/chosen": 0.2171139419078827,
-      "eval_rewards/margins": 0.030863292515277863,
-      "eval_rewards/rejected": 0.18625062704086304,
-      "eval_runtime": 146.2598,
-      "eval_samples_per_second": 2.345,
-      "eval_steps_per_second": 0.294,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -2.0422446727752686,
-      "logits/rejected": -2.0450665950775146,
-      "logps/chosen": -31.823333740234375,
-      "logps/rejected": -34.91590118408203,
-      "loss": 0.3378,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.7383630871772766,
-      "rewards/margins": 0.8121965527534485,
-      "rewards/rejected": -0.07383350282907486,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.8828691244125366,
-      "logits/rejected": -1.8815042972564697,
-      "logps/chosen": -27.965286254882812,
-      "logps/rejected": -32.40424346923828,
-      "loss": 0.3525,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6613627672195435,
-      "rewards/margins": 0.7133124470710754,
-      "rewards/rejected": -0.05194968730211258,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.8411157131195068,
-      "logits/rejected": -1.8340963125228882,
-      "logps/chosen": -32.586795806884766,
-      "logps/rejected": -34.0372428894043,
-      "loss": 0.334,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.8923752903938293,
-      "rewards/margins": 0.7675577402114868,
-      "rewards/rejected": 0.1248176246881485,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -2.01190185546875,
-      "logits/rejected": -2.006816864013672,
-      "logps/chosen": -30.412521362304688,
-      "logps/rejected": -32.05153274536133,
-      "loss": 0.3237,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.8224636316299438,
-      "rewards/margins": 0.8728159666061401,
-      "rewards/rejected": -0.05035240575671196,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -1.8965171575546265,
-      "logits/rejected": -1.9006315469741821,
-      "logps/chosen": -32.99296569824219,
-      "logps/rejected": -32.713218688964844,
-      "loss": 0.302,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.913729190826416,
-      "rewards/margins": 0.9884780049324036,
-      "rewards/rejected": -0.07474879175424576,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -1.8998197317123413,
-      "logits/rejected": -1.8941733837127686,
-      "logps/chosen": -33.83064651489258,
-      "logps/rejected": -35.21913146972656,
-      "loss": 0.2931,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9533284902572632,
-      "rewards/margins": 1.115350365638733,
-      "rewards/rejected": -0.16202178597450256,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.9277269840240479,
-      "logits/rejected": -1.9273641109466553,
-      "logps/chosen": -32.60927963256836,
-      "logps/rejected": -34.12434005737305,
-      "loss": 0.3206,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.9354284405708313,
-      "rewards/margins": 0.8987458348274231,
-      "rewards/rejected": 0.0366826057434082,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -1.9775463342666626,
-      "logits/rejected": -1.9590513706207275,
-      "logps/chosen": -32.508819580078125,
-      "logps/rejected": -33.2094612121582,
-      "loss": 0.2906,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.9911691546440125,
-      "rewards/margins": 1.07485830783844,
-      "rewards/rejected": -0.08368916064500809,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9436886310577393,
-      "logits/rejected": -1.9431581497192383,
-      "logps/chosen": -30.075321197509766,
-      "logps/rejected": -31.215450286865234,
-      "loss": 0.3167,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.9215895533561707,
-      "rewards/margins": 0.9032338261604309,
-      "rewards/rejected": 0.018355702981352806,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.9060132503509521,
-      "logits/rejected": -1.8992029428482056,
-      "logps/chosen": -30.9982852935791,
-      "logps/rejected": -34.8018798828125,
-      "loss": 0.2906,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.0031068325042725,
-      "rewards/margins": 1.1081880331039429,
-      "rewards/rejected": -0.10508137941360474,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.1881351470947266,
-      "eval_logits/rejected": -2.183356761932373,
-      "eval_logps/chosen": -33.62051010131836,
-      "eval_logps/rejected": -37.196380615234375,
-      "eval_loss": 0.48696067929267883,
-      "eval_rewards/accuracies": 0.5921927094459534,
-      "eval_rewards/chosen": 0.24842412769794464,
-      "eval_rewards/margins": 0.0562782883644104,
-      "eval_rewards/rejected": 0.19214583933353424,
-      "eval_runtime": 145.95,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.9005699157714844,
-      "logits/rejected": -1.9044411182403564,
-      "logps/chosen": -27.744693756103516,
-      "logps/rejected": -33.00718688964844,
-      "loss": 0.3144,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.8580785989761353,
-      "rewards/margins": 0.9468171000480652,
-      "rewards/rejected": -0.08873851597309113,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -1.907922387123108,
-      "logits/rejected": -1.9184629917144775,
-      "logps/chosen": -31.68210220336914,
-      "logps/rejected": -30.8319091796875,
-      "loss": 0.3058,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.9481385946273804,
-      "rewards/margins": 1.056273102760315,
-      "rewards/rejected": -0.1081344336271286,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.8528549671173096,
-      "logits/rejected": -1.8505821228027344,
-      "logps/chosen": -30.023778915405273,
-      "logps/rejected": -30.290355682373047,
-      "loss": 0.2945,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.9859828948974609,
-      "rewards/margins": 1.054137945175171,
-      "rewards/rejected": -0.06815493851900101,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.9326980113983154,
-      "logits/rejected": -1.925724744796753,
-      "logps/chosen": -32.48560333251953,
-      "logps/rejected": -33.21514892578125,
-      "loss": 0.2771,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.0483348369598389,
-      "rewards/margins": 1.1669293642044067,
-      "rewards/rejected": -0.11859482526779175,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.8988561630249023,
-      "logits/rejected": -1.905217170715332,
-      "logps/chosen": -30.224956512451172,
-      "logps/rejected": -32.778907775878906,
-      "loss": 0.3108,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.9169367551803589,
-      "rewards/margins": 0.929042637348175,
-      "rewards/rejected": -0.012105767615139484,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -1.9703216552734375,
-      "logits/rejected": -1.9476426839828491,
-      "logps/chosen": -29.846935272216797,
-      "logps/rejected": -33.04885482788086,
-      "loss": 0.3058,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.9543794393539429,
-      "rewards/margins": 0.9860580563545227,
-      "rewards/rejected": -0.031678564846515656,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.9047954082489014,
-      "logits/rejected": -1.899834394454956,
-      "logps/chosen": -30.691064834594727,
-      "logps/rejected": -32.055702209472656,
-      "loss": 0.305,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.0354000329971313,
-      "rewards/margins": 0.9962730407714844,
-      "rewards/rejected": 0.039127081632614136,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -1.9474509954452515,
-      "logits/rejected": -1.9493205547332764,
-      "logps/chosen": -32.68132781982422,
-      "logps/rejected": -35.09022903442383,
-      "loss": 0.2831,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0166804790496826,
-      "rewards/margins": 1.19026780128479,
-      "rewards/rejected": -0.17358721792697906,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.929598093032837,
-      "logits/rejected": -1.9289600849151611,
-      "logps/chosen": -28.859294891357422,
-      "logps/rejected": -31.535837173461914,
-      "loss": 0.2904,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.0396578311920166,
-      "rewards/margins": 1.084177017211914,
-      "rewards/rejected": -0.04451901838183403,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.9405543804168701,
-      "logits/rejected": -1.93850576877594,
-      "logps/chosen": -31.257436752319336,
-      "logps/rejected": -32.223670959472656,
-      "loss": 0.3014,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.0445080995559692,
-      "rewards/margins": 0.9987362623214722,
-      "rewards/rejected": 0.045771874487400055,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.174114942550659,
-      "eval_logits/rejected": -2.1693577766418457,
-      "eval_logps/chosen": -33.596221923828125,
-      "eval_logps/rejected": -37.230098724365234,
-      "eval_loss": 0.4795546531677246,
-      "eval_rewards/accuracies": 0.579734206199646,
-      "eval_rewards/chosen": 0.26299843192100525,
-      "eval_rewards/margins": 0.09108825773000717,
-      "eval_rewards/rejected": 0.17191016674041748,
-      "eval_runtime": 145.976,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -1.9819376468658447,
-      "logits/rejected": -1.9884719848632812,
-      "logps/chosen": -30.554279327392578,
-      "logps/rejected": -32.0571403503418,
-      "loss": 0.3038,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.0098621845245361,
-      "rewards/margins": 1.0062358379364014,
-      "rewards/rejected": 0.0036262169014662504,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -1.9370014667510986,
-      "logits/rejected": -1.9347403049468994,
-      "logps/chosen": -32.8254508972168,
-      "logps/rejected": -31.027420043945312,
-      "loss": 0.2776,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.1582577228546143,
-      "rewards/margins": 1.1834052801132202,
-      "rewards/rejected": -0.025147801265120506,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.8845012187957764,
-      "logits/rejected": -1.8778787851333618,
-      "logps/chosen": -33.42643737792969,
-      "logps/rejected": -32.37970733642578,
-      "loss": 0.2689,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.2226732969284058,
-      "rewards/margins": 1.2361730337142944,
-      "rewards/rejected": -0.013499641790986061,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.8900282382965088,
-      "logits/rejected": -1.8962434530258179,
-      "logps/chosen": -32.14881134033203,
-      "logps/rejected": -33.176612854003906,
-      "loss": 0.2981,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.098296880722046,
-      "rewards/margins": 1.0601081848144531,
-      "rewards/rejected": 0.03818872570991516,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -1.9740774631500244,
-      "logits/rejected": -1.9881250858306885,
-      "logps/chosen": -30.15896987915039,
-      "logps/rejected": -31.947906494140625,
-      "loss": 0.3021,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.0160971879959106,
-      "rewards/margins": 0.9862273335456848,
-      "rewards/rejected": 0.02987002767622471,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.8505609035491943,
-      "logits/rejected": -1.847583532333374,
-      "logps/chosen": -31.669322967529297,
-      "logps/rejected": -34.97021484375,
-      "loss": 0.2316,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.3521816730499268,
-      "rewards/margins": 1.5141586065292358,
-      "rewards/rejected": -0.16197696328163147,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.9402939081192017,
-      "logits/rejected": -1.9406039714813232,
-      "logps/chosen": -29.579147338867188,
-      "logps/rejected": -33.99906921386719,
-      "loss": 0.2843,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.0238673686981201,
-      "rewards/margins": 1.1274796724319458,
-      "rewards/rejected": -0.10361222922801971,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.8894329071044922,
-      "logits/rejected": -1.8864387273788452,
-      "logps/chosen": -29.1514892578125,
-      "logps/rejected": -32.90578079223633,
-      "loss": 0.2983,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.0325144529342651,
-      "rewards/margins": 1.0853002071380615,
-      "rewards/rejected": -0.05278577655553818,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.9213037490844727,
-      "logits/rejected": -1.921303391456604,
-      "logps/chosen": -27.972620010375977,
-      "logps/rejected": -30.834762573242188,
-      "loss": 0.2479,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.314284086227417,
-      "rewards/margins": 1.4128423929214478,
-      "rewards/rejected": -0.09855831414461136,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.8346879482269287,
-      "logits/rejected": -1.838030219078064,
-      "logps/chosen": -32.32703399658203,
-      "logps/rejected": -31.50747299194336,
-      "loss": 0.2776,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.2908637523651123,
-      "rewards/margins": 1.316535234451294,
-      "rewards/rejected": -0.025671422481536865,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.1672425270080566,
-      "eval_logits/rejected": -2.162508487701416,
-      "eval_logps/chosen": -33.64436340332031,
-      "eval_logps/rejected": -37.25758743286133,
-      "eval_loss": 0.4824546277523041,
-      "eval_rewards/accuracies": 0.5768272280693054,
-      "eval_rewards/chosen": 0.23411329090595245,
-      "eval_rewards/margins": 0.07869596779346466,
-      "eval_rewards/rejected": 0.155417338013649,
-      "eval_runtime": 145.94,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -1.9178850650787354,
-      "logits/rejected": -1.9089438915252686,
-      "logps/chosen": -34.47451400756836,
-      "logps/rejected": -31.956844329833984,
-      "loss": 0.2673,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.2225029468536377,
-      "rewards/margins": 1.2531225681304932,
-      "rewards/rejected": -0.030619746074080467,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.9318691492080688,
-      "logits/rejected": -1.9329137802124023,
-      "logps/chosen": -34.28239440917969,
-      "logps/rejected": -33.12510681152344,
-      "loss": 0.2565,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.291794776916504,
-      "rewards/margins": 1.3212449550628662,
-      "rewards/rejected": -0.02945004776120186,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -1.9492158889770508,
-      "logits/rejected": -1.9468295574188232,
-      "logps/chosen": -30.499465942382812,
-      "logps/rejected": -33.04973220825195,
-      "loss": 0.2684,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.2383159399032593,
-      "rewards/margins": 1.270018219947815,
-      "rewards/rejected": -0.03170236200094223,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.933258056640625,
-      "logits/rejected": -1.938474416732788,
-      "logps/chosen": -29.6435546875,
-      "logps/rejected": -33.32258224487305,
-      "loss": 0.2485,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.2853031158447266,
-      "rewards/margins": 1.38596510887146,
-      "rewards/rejected": -0.10066213458776474,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.8792998790740967,
-      "logits/rejected": -1.8781516551971436,
-      "logps/chosen": -31.056386947631836,
-      "logps/rejected": -34.74039840698242,
-      "loss": 0.274,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.2411561012268066,
-      "rewards/margins": 1.2738196849822998,
-      "rewards/rejected": -0.032663825899362564,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.754612684249878,
-      "logits/rejected": -1.7496144771575928,
-      "logps/chosen": -33.715721130371094,
-      "logps/rejected": -35.355167388916016,
-      "loss": 0.253,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.3696316480636597,
-      "rewards/margins": 1.4576833248138428,
-      "rewards/rejected": -0.08805163949728012,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.8957417011260986,
-      "logits/rejected": -1.8994983434677124,
-      "logps/chosen": -33.10436248779297,
-      "logps/rejected": -33.5843505859375,
-      "loss": 0.2679,
-      "rewards/accuracies": 0.8833333849906921,
-      "rewards/chosen": 1.2655742168426514,
-      "rewards/margins": 1.2592796087265015,
-      "rewards/rejected": 0.006294435355812311,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.8399032354354858,
-      "logits/rejected": -1.8381221294403076,
-      "logps/chosen": -31.176462173461914,
-      "logps/rejected": -35.09248733520508,
-      "loss": 0.1728,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.6350797414779663,
-      "rewards/margins": 2.047778606414795,
-      "rewards/rejected": -0.412698894739151,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.8872671127319336,
-      "logits/rejected": -1.8859704732894897,
-      "logps/chosen": -30.6798095703125,
-      "logps/rejected": -33.45615005493164,
-      "loss": 0.1882,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6175569295883179,
-      "rewards/margins": 1.8393020629882812,
-      "rewards/rejected": -0.2217450588941574,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.8201385736465454,
-      "logits/rejected": -1.8143587112426758,
-      "logps/chosen": -31.758071899414062,
-      "logps/rejected": -31.750629425048828,
-      "loss": 0.201,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5750051736831665,
-      "rewards/margins": 1.7468246221542358,
-      "rewards/rejected": -0.17181925475597382,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.168851375579834,
-      "eval_logits/rejected": -2.164125919342041,
-      "eval_logps/chosen": -33.594791412353516,
-      "eval_logps/rejected": -37.25070571899414,
-      "eval_loss": 0.4766196310520172,
-      "eval_rewards/accuracies": 0.5913621187210083,
-      "eval_rewards/chosen": 0.2638549208641052,
-      "eval_rewards/margins": 0.10430653393268585,
-      "eval_rewards/rejected": 0.15954840183258057,
-      "eval_runtime": 145.9506,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.916377067565918,
-      "logits/rejected": -1.923803687095642,
-      "logps/chosen": -29.560104370117188,
-      "logps/rejected": -34.730369567871094,
-      "loss": 0.2043,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5408909320831299,
-      "rewards/margins": 1.8256099224090576,
-      "rewards/rejected": -0.2847188711166382,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.8899390697479248,
-      "logits/rejected": -1.8926613330841064,
-      "logps/chosen": -30.270843505859375,
-      "logps/rejected": -33.329734802246094,
-      "loss": 0.2432,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.3801411390304565,
-      "rewards/margins": 1.4400979280471802,
-      "rewards/rejected": -0.05995672196149826,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.7909173965454102,
-      "logits/rejected": -1.7950553894042969,
-      "logps/chosen": -30.025737762451172,
-      "logps/rejected": -35.363929748535156,
-      "loss": 0.1984,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.5394319295883179,
-      "rewards/margins": 1.8116801977157593,
-      "rewards/rejected": -0.2722483277320862,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.8557007312774658,
-      "logits/rejected": -1.851709008216858,
-      "logps/chosen": -30.498058319091797,
-      "logps/rejected": -33.070526123046875,
-      "loss": 0.2262,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.482353925704956,
-      "rewards/margins": 1.527881145477295,
-      "rewards/rejected": -0.04552716761827469,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.9369919300079346,
-      "logits/rejected": -1.9389537572860718,
-      "logps/chosen": -30.1755428314209,
-      "logps/rejected": -36.147056579589844,
-      "loss": 0.1878,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.5598775148391724,
-      "rewards/margins": 1.850951910018921,
-      "rewards/rejected": -0.29107466340065,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -1.7436918020248413,
-      "logits/rejected": -1.738072156906128,
-      "logps/chosen": -33.61563491821289,
-      "logps/rejected": -33.156700134277344,
-      "loss": 0.2065,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.5996261835098267,
-      "rewards/margins": 1.7585023641586304,
-      "rewards/rejected": -0.15887616574764252,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.8033043146133423,
-      "logits/rejected": -1.803333044052124,
-      "logps/chosen": -33.05567169189453,
-      "logps/rejected": -35.33915328979492,
-      "loss": 0.1857,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.6722819805145264,
-      "rewards/margins": 1.9680156707763672,
-      "rewards/rejected": -0.29573339223861694,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.850695013999939,
-      "logits/rejected": -1.839464783668518,
-      "logps/chosen": -31.547618865966797,
-      "logps/rejected": -33.39630126953125,
-      "loss": 0.1969,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.535375714302063,
-      "rewards/margins": 1.8199365139007568,
-      "rewards/rejected": -0.2845609188079834,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.9542624950408936,
-      "logits/rejected": -1.9587194919586182,
-      "logps/chosen": -30.739770889282227,
-      "logps/rejected": -33.01618194580078,
-      "loss": 0.1947,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5540999174118042,
-      "rewards/margins": 1.9019180536270142,
-      "rewards/rejected": -0.34781819581985474,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.8733972311019897,
-      "logits/rejected": -1.8663629293441772,
-      "logps/chosen": -31.99909019470215,
-      "logps/rejected": -33.01835632324219,
-      "loss": 0.1721,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.7129936218261719,
-      "rewards/margins": 2.0827279090881348,
-      "rewards/rejected": -0.3697340786457062,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.156324625015259,
-      "eval_logits/rejected": -2.151609420776367,
-      "eval_logps/chosen": -33.62693786621094,
-      "eval_logps/rejected": -37.30033493041992,
-      "eval_loss": 0.4748935401439667,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": 0.2445676475763321,
-      "eval_rewards/margins": 0.11479608714580536,
-      "eval_rewards/rejected": 0.12977157533168793,
-      "eval_runtime": 145.9673,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.864915132522583,
-      "logits/rejected": -1.8653583526611328,
-      "logps/chosen": -28.593013763427734,
-      "logps/rejected": -35.23602294921875,
-      "loss": 0.1986,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.5444881916046143,
-      "rewards/margins": 1.8722877502441406,
-      "rewards/rejected": -0.3277994692325592,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.8232961893081665,
-      "logits/rejected": -1.815349817276001,
-      "logps/chosen": -29.518482208251953,
-      "logps/rejected": -34.129737854003906,
-      "loss": 0.2142,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.3922770023345947,
-      "rewards/margins": 1.7272971868515015,
-      "rewards/rejected": -0.3350200951099396,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -1.9893302917480469,
-      "logits/rejected": -1.9897470474243164,
-      "logps/chosen": -27.58512306213379,
-      "logps/rejected": -33.1852912902832,
-      "loss": 0.233,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.4471487998962402,
-      "rewards/margins": 1.6394065618515015,
-      "rewards/rejected": -0.1922578513622284,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.8695605993270874,
-      "logits/rejected": -1.8790569305419922,
-      "logps/chosen": -31.543869018554688,
-      "logps/rejected": -33.519187927246094,
-      "loss": 0.1757,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.7287185192108154,
-      "rewards/margins": 2.046445369720459,
-      "rewards/rejected": -0.31772682070732117,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.9113248586654663,
-      "logits/rejected": -1.9151248931884766,
-      "logps/chosen": -28.23110580444336,
-      "logps/rejected": -32.094093322753906,
-      "loss": 0.2093,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.4499690532684326,
-      "rewards/margins": 1.664777398109436,
-      "rewards/rejected": -0.21480807662010193,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.9074852466583252,
-      "logits/rejected": -1.9036372900009155,
-      "logps/chosen": -31.310367584228516,
-      "logps/rejected": -34.39436721801758,
-      "loss": 0.1936,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6589105129241943,
-      "rewards/margins": 1.951696753501892,
-      "rewards/rejected": -0.2927861213684082,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.8242709636688232,
-      "logits/rejected": -1.8268591165542603,
-      "logps/chosen": -30.07489585876465,
-      "logps/rejected": -35.65007400512695,
-      "loss": 0.1939,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.642709493637085,
-      "rewards/margins": 1.9237987995147705,
-      "rewards/rejected": -0.28108924627304077,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.8587324619293213,
-      "logits/rejected": -1.8523610830307007,
-      "logps/chosen": -32.797035217285156,
-      "logps/rejected": -36.79326248168945,
-      "loss": 0.2006,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.564818263053894,
-      "rewards/margins": 1.8836040496826172,
-      "rewards/rejected": -0.3187856078147888,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.8334388732910156,
-      "logits/rejected": -1.8283811807632446,
-      "logps/chosen": -28.48984718322754,
-      "logps/rejected": -36.4734001159668,
-      "loss": 0.2097,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.5078611373901367,
-      "rewards/margins": 1.7979707717895508,
-      "rewards/rejected": -0.2901095449924469,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.8117061853408813,
-      "logits/rejected": -1.8011939525604248,
-      "logps/chosen": -29.4431095123291,
-      "logps/rejected": -29.787906646728516,
-      "loss": 0.2259,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.3833496570587158,
-      "rewards/margins": 1.4962539672851562,
-      "rewards/rejected": -0.11290428787469864,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.1527934074401855,
-      "eval_logits/rejected": -2.1480958461761475,
-      "eval_logps/chosen": -33.62074661254883,
-      "eval_logps/rejected": -37.30718994140625,
-      "eval_loss": 0.4736105799674988,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": 0.24828366935253143,
-      "eval_rewards/margins": 0.12262485176324844,
-      "eval_rewards/rejected": 0.12565883994102478,
-      "eval_runtime": 145.9544,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.9208866357803345,
-      "logits/rejected": -1.9203205108642578,
-      "logps/chosen": -29.327899932861328,
-      "logps/rejected": -30.58652114868164,
-      "loss": 0.2156,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.4977772235870361,
-      "rewards/margins": 1.6946022510528564,
-      "rewards/rejected": -0.1968248337507248,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.9610830545425415,
-      "logits/rejected": -1.9618806838989258,
-      "logps/chosen": -29.420618057250977,
-      "logps/rejected": -31.693790435791016,
-      "loss": 0.1936,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.5137760639190674,
-      "rewards/margins": 1.830562949180603,
-      "rewards/rejected": -0.31678706407546997,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -1.9309966564178467,
-      "logits/rejected": -1.934212327003479,
-      "logps/chosen": -29.75461769104004,
-      "logps/rejected": -28.163843154907227,
-      "loss": 0.2199,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5152724981307983,
-      "rewards/margins": 1.6233495473861694,
-      "rewards/rejected": -0.10807719081640244,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.9402587413787842,
-      "logits/rejected": -1.9325793981552124,
-      "logps/chosen": -28.6353702545166,
-      "logps/rejected": -34.016441345214844,
-      "loss": 0.1958,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5255171060562134,
-      "rewards/margins": 1.8352447748184204,
-      "rewards/rejected": -0.30972760915756226,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.8774486780166626,
-      "logits/rejected": -1.888318419456482,
-      "logps/chosen": -30.820209503173828,
-      "logps/rejected": -32.5506706237793,
-      "loss": 0.1925,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.6397380828857422,
-      "rewards/margins": 1.8231319189071655,
-      "rewards/rejected": -0.18339397013187408,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.9557583332061768,
-      "logits/rejected": -1.9465582370758057,
-      "logps/chosen": -30.896408081054688,
-      "logps/rejected": -36.84226608276367,
-      "loss": 0.2111,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.5626221895217896,
-      "rewards/margins": 1.8691476583480835,
-      "rewards/rejected": -0.3065256178379059,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.8635246753692627,
-      "logits/rejected": -1.8655227422714233,
-      "logps/chosen": -33.10884475708008,
-      "logps/rejected": -33.1629753112793,
-      "loss": 0.1786,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.7282660007476807,
-      "rewards/margins": 2.0073320865631104,
-      "rewards/rejected": -0.2790663540363312,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.993586540222168,
-      "logits/rejected": -1.9946495294570923,
-      "logps/chosen": -30.365489959716797,
-      "logps/rejected": -34.76932907104492,
-      "loss": 0.1816,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.6152902841567993,
-      "rewards/margins": 2.0074784755706787,
-      "rewards/rejected": -0.39218825101852417,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -1.9641876220703125,
-      "logits/rejected": -1.9684619903564453,
-      "logps/chosen": -28.4173583984375,
-      "logps/rejected": -33.92015838623047,
-      "loss": 0.2133,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.4855743646621704,
-      "rewards/margins": 1.7089710235595703,
-      "rewards/rejected": -0.22339674830436707,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.9531924724578857,
-      "logits/rejected": -1.958428144454956,
-      "logps/chosen": -31.164684295654297,
-      "logps/rejected": -30.430471420288086,
-      "loss": 0.2405,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.4425894021987915,
-      "rewards/margins": 1.459869623184204,
-      "rewards/rejected": -0.01728021167218685,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.1522130966186523,
-      "eval_logits/rejected": -2.147509813308716,
-      "eval_logps/chosen": -33.62826156616211,
-      "eval_logps/rejected": -37.31184005737305,
-      "eval_loss": 0.4740113317966461,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": 0.24377261102199554,
-      "eval_rewards/margins": 0.1209048256278038,
-      "eval_rewards/rejected": 0.12286783009767532,
-      "eval_runtime": 145.9558,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -1.9431841373443604,
-      "logits/rejected": -1.9432331323623657,
-      "logps/chosen": -30.208242416381836,
-      "logps/rejected": -31.27047348022461,
-      "loss": 0.1964,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.6391448974609375,
-      "rewards/margins": 1.7918792963027954,
-      "rewards/rejected": -0.15273427963256836,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.788575530052185,
-      "logits/rejected": -1.7902095317840576,
-      "logps/chosen": -31.8612117767334,
-      "logps/rejected": -33.35196304321289,
-      "loss": 0.2123,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.6222608089447021,
-      "rewards/margins": 1.8086793422698975,
-      "rewards/rejected": -0.18641847372055054,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.8251508474349976,
-      "logits/rejected": -1.8224670886993408,
-      "logps/chosen": -28.645374298095703,
-      "logps/rejected": -31.369842529296875,
-      "loss": 0.2402,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.4198086261749268,
-      "rewards/margins": 1.542269229888916,
-      "rewards/rejected": -0.12246046215295792,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -1.9626710414886475,
-      "logits/rejected": -1.957177758216858,
-      "logps/chosen": -30.216720581054688,
-      "logps/rejected": -33.21830368041992,
-      "loss": 0.1933,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6938445568084717,
-      "rewards/margins": 1.8524214029312134,
-      "rewards/rejected": -0.15857680141925812,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.8174095153808594,
-      "logits/rejected": -1.807636022567749,
-      "logps/chosen": -30.805782318115234,
-      "logps/rejected": -31.54193687438965,
-      "loss": 0.2166,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5875717401504517,
-      "rewards/margins": 1.737291693687439,
-      "rewards/rejected": -0.14972007274627686,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.8910198211669922,
-      "logits/rejected": -1.8909051418304443,
-      "logps/chosen": -30.962215423583984,
-      "logps/rejected": -31.599491119384766,
-      "loss": 0.1905,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.6317226886749268,
-      "rewards/margins": 1.8705470561981201,
-      "rewards/rejected": -0.23882456123828888,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -1.96527099609375,
-      "logits/rejected": -1.9599376916885376,
-      "logps/chosen": -30.861770629882812,
-      "logps/rejected": -33.239105224609375,
-      "loss": 0.1977,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.5985205173492432,
-      "rewards/margins": 1.7772538661956787,
-      "rewards/rejected": -0.17873358726501465,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.8232942819595337,
-      "logits/rejected": -1.8309834003448486,
-      "logps/chosen": -30.7927188873291,
-      "logps/rejected": -32.76128387451172,
-      "loss": 0.2011,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.5241663455963135,
-      "rewards/margins": 1.7640721797943115,
-      "rewards/rejected": -0.23990590870380402,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.8941972255706787,
-      "logits/rejected": -1.8883616924285889,
-      "logps/chosen": -29.16994857788086,
-      "logps/rejected": -33.72978973388672,
-      "loss": 0.1689,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8021999597549438,
-      "rewards/margins": 2.0700056552886963,
-      "rewards/rejected": -0.26780563592910767,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.8379895687103271,
-      "logits/rejected": -1.8400685787200928,
-      "logps/chosen": -30.718734741210938,
-      "logps/rejected": -33.35673141479492,
-      "loss": 0.1793,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.6630252599716187,
-      "rewards/margins": 2.0210342407226562,
-      "rewards/rejected": -0.35800907015800476,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.151599884033203,
-      "eval_logits/rejected": -2.1469039916992188,
-      "eval_logps/chosen": -33.627723693847656,
-      "eval_logps/rejected": -37.30845642089844,
-      "eval_loss": 0.47458338737487793,
-      "eval_rewards/accuracies": 0.5685215592384338,
-      "eval_rewards/chosen": 0.24409595131874084,
-      "eval_rewards/margins": 0.11919664591550827,
-      "eval_rewards/rejected": 0.12489929050207138,
-      "eval_runtime": 145.9719,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.7688223123550415,
-      "logits/rejected": -1.7678744792938232,
-      "logps/chosen": -33.03181076049805,
-      "logps/rejected": -33.192596435546875,
-      "loss": 0.1724,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.8124043941497803,
-      "rewards/margins": 2.102344036102295,
-      "rewards/rejected": -0.2899395823478699,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.9436089992523193,
-      "logits/rejected": -1.9377737045288086,
-      "logps/chosen": -28.891845703125,
-      "logps/rejected": -33.52208709716797,
-      "loss": 0.1785,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.6871049404144287,
-      "rewards/margins": 2.0732340812683105,
-      "rewards/rejected": -0.38612908124923706,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.79269278049469,
-      "logits/rejected": -1.7812188863754272,
-      "logps/chosen": -31.352039337158203,
-      "logps/rejected": -33.33822250366211,
-      "loss": 0.1708,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.820186972618103,
-      "rewards/margins": 2.086826801300049,
-      "rewards/rejected": -0.2666396200656891,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.8849557638168335,
-      "logits/rejected": -1.878165602684021,
-      "logps/chosen": -28.257898330688477,
-      "logps/rejected": -29.390756607055664,
-      "loss": 0.2132,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.4654014110565186,
-      "rewards/margins": 1.7793182134628296,
-      "rewards/rejected": -0.31391677260398865,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.9139467477798462,
-      "logits/rejected": -1.9205329418182373,
-      "logps/chosen": -32.439273834228516,
-      "logps/rejected": -31.840866088867188,
-      "loss": 0.1773,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.7736711502075195,
-      "rewards/margins": 1.9672033786773682,
-      "rewards/rejected": -0.19353218376636505,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.8897371292114258,
-      "logits/rejected": -1.890363097190857,
-      "logps/chosen": -30.296092987060547,
-      "logps/rejected": -33.34063720703125,
-      "loss": 0.1781,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.639479637145996,
-      "rewards/margins": 1.9549386501312256,
-      "rewards/rejected": -0.3154590129852295,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.8287382125854492,
-      "logits/rejected": -1.8283493518829346,
-      "logps/chosen": -30.095571517944336,
-      "logps/rejected": -34.024879455566406,
-      "loss": 0.1635,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.7971582412719727,
-      "rewards/margins": 2.0997211933135986,
-      "rewards/rejected": -0.30256301164627075,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.7129100561141968,
-      "logits/rejected": -1.7082204818725586,
-      "logps/chosen": -32.2062873840332,
-      "logps/rejected": -31.325271606445312,
-      "loss": 0.1593,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.818695068359375,
-      "rewards/margins": 2.0579652786254883,
-      "rewards/rejected": -0.2392701357603073,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.9185450077056885,
-      "logits/rejected": -1.903119444847107,
-      "logps/chosen": -31.7047119140625,
-      "logps/rejected": -35.49399948120117,
-      "loss": 0.1718,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.5857194662094116,
-      "rewards/margins": 2.022928237915039,
-      "rewards/rejected": -0.43720903992652893,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.8624118566513062,
-      "logits/rejected": -1.8672168254852295,
-      "logps/chosen": -30.548635482788086,
-      "logps/rejected": -36.78308868408203,
-      "loss": 0.1633,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.781390905380249,
-      "rewards/margins": 2.1893134117126465,
-      "rewards/rejected": -0.4079226553440094,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.151759624481201,
-      "eval_logits/rejected": -2.14705753326416,
-      "eval_logps/chosen": -33.628997802734375,
-      "eval_logps/rejected": -37.31074523925781,
-      "eval_loss": 0.4743698537349701,
-      "eval_rewards/accuracies": 0.6009136438369751,
-      "eval_rewards/chosen": 0.24333173036575317,
-      "eval_rewards/margins": 0.11980798840522766,
-      "eval_rewards/rejected": 0.12352374196052551,
-      "eval_runtime": 145.9607,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.8351796865463257,
-      "logits/rejected": -1.8353471755981445,
-      "logps/chosen": -33.059471130371094,
-      "logps/rejected": -37.527793884277344,
-      "loss": 0.1742,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.9107275009155273,
-      "rewards/margins": 2.176933765411377,
-      "rewards/rejected": -0.26620611548423767,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -1.9216110706329346,
-      "logits/rejected": -1.9202417135238647,
-      "logps/chosen": -29.405132293701172,
-      "logps/rejected": -31.430126190185547,
-      "loss": 0.2013,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6132328510284424,
-      "rewards/margins": 1.8992938995361328,
-      "rewards/rejected": -0.2860611379146576,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.8282123804092407,
-      "logits/rejected": -1.829431176185608,
-      "logps/chosen": -31.4184627532959,
-      "logps/rejected": -34.05662155151367,
-      "loss": 0.181,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.773472785949707,
-      "rewards/margins": 2.0182414054870605,
-      "rewards/rejected": -0.24476829171180725,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.8872970342636108,
-      "logits/rejected": -1.8860971927642822,
-      "logps/chosen": -28.597339630126953,
-      "logps/rejected": -32.5714225769043,
-      "loss": 0.1828,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.5607964992523193,
-      "rewards/margins": 1.908465027809143,
-      "rewards/rejected": -0.34766843914985657,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.9047062397003174,
-      "logits/rejected": -1.890319585800171,
-      "logps/chosen": -31.581689834594727,
-      "logps/rejected": -36.782928466796875,
-      "loss": 0.1839,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.6409885883331299,
-      "rewards/margins": 1.9520866870880127,
-      "rewards/rejected": -0.31109803915023804,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.9034063816070557,
-      "logits/rejected": -1.9158471822738647,
-      "logps/chosen": -29.19135093688965,
-      "logps/rejected": -32.5649299621582,
-      "loss": 0.1788,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.726271390914917,
-      "rewards/margins": 1.921381950378418,
-      "rewards/rejected": -0.19511063396930695,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -1.9984302520751953,
-      "logits/rejected": -1.9977290630340576,
-      "logps/chosen": -29.15370750427246,
-      "logps/rejected": -36.60697555541992,
-      "loss": 0.1649,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.7260326147079468,
-      "rewards/margins": 2.191688060760498,
-      "rewards/rejected": -0.46565547585487366,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.863462209701538,
-      "logits/rejected": -1.8711704015731812,
-      "logps/chosen": -31.19002914428711,
-      "logps/rejected": -37.98977279663086,
-      "loss": 0.183,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.7942688465118408,
-      "rewards/margins": 2.118346929550171,
-      "rewards/rejected": -0.3240783214569092,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.8185231685638428,
-      "logits/rejected": -1.8144798278808594,
-      "logps/chosen": -31.748987197875977,
-      "logps/rejected": -31.824060440063477,
-      "loss": 0.1855,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.8109363317489624,
-      "rewards/margins": 1.9469363689422607,
-      "rewards/rejected": -0.1359998881816864,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.9791545867919922,
-      "logits/rejected": -1.9766286611557007,
-      "logps/chosen": -28.732391357421875,
-      "logps/rejected": -32.56377410888672,
-      "loss": 0.202,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5384950637817383,
-      "rewards/margins": 1.790077567100525,
-      "rewards/rejected": -0.25158247351646423,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.1518642902374268,
-      "eval_logits/rejected": -2.147162437438965,
-      "eval_logps/chosen": -33.62628173828125,
-      "eval_logps/rejected": -37.30340576171875,
-      "eval_loss": 0.4747585952281952,
-      "eval_rewards/accuracies": 0.5830564498901367,
-      "eval_rewards/chosen": 0.24496087431907654,
-      "eval_rewards/margins": 0.11703355610370636,
-      "eval_rewards/rejected": 0.1279272884130478,
-      "eval_runtime": 145.9829,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -1.9209346771240234,
-      "logits/rejected": -1.916852355003357,
-      "logps/chosen": -30.77335548400879,
-      "logps/rejected": -29.9306697845459,
-      "loss": 0.1787,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.7215063571929932,
-      "rewards/margins": 1.947231650352478,
-      "rewards/rejected": -0.22572548687458038,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.878213882446289,
-      "logits/rejected": -1.8793401718139648,
-      "logps/chosen": -30.57177734375,
-      "logps/rejected": -35.1507682800293,
-      "loss": 0.1981,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6758613586425781,
-      "rewards/margins": 1.8119198083877563,
-      "rewards/rejected": -0.13605818152427673,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.8281654119491577,
-      "logits/rejected": -1.8215233087539673,
-      "logps/chosen": -31.386241912841797,
-      "logps/rejected": -37.23295211791992,
-      "loss": 0.1849,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.653316855430603,
-      "rewards/margins": 1.9794448614120483,
-      "rewards/rejected": -0.3261282444000244,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.8422043323516846,
-      "logits/rejected": -1.836806297302246,
-      "logps/chosen": -26.592382431030273,
-      "logps/rejected": -32.13856887817383,
-      "loss": 0.1895,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.6325156688690186,
-      "rewards/margins": 1.8772624731063843,
-      "rewards/rejected": -0.2447468340396881,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.8666261434555054,
-      "logits/rejected": -1.8564945459365845,
-      "logps/chosen": -30.06821632385254,
-      "logps/rejected": -36.11262130737305,
-      "loss": 0.1911,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.4479871988296509,
-      "rewards/margins": 1.9443438053131104,
-      "rewards/rejected": -0.49635666608810425,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.8395572900772095,
-      "logits/rejected": -1.836773157119751,
-      "logps/chosen": -30.69950294494629,
-      "logps/rejected": -31.030874252319336,
-      "loss": 0.21,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5661640167236328,
-      "rewards/margins": 1.6501954793930054,
-      "rewards/rejected": -0.08403147757053375,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.9007151126861572,
-      "logits/rejected": -1.9071681499481201,
-      "logps/chosen": -29.030803680419922,
-      "logps/rejected": -33.39077377319336,
-      "loss": 0.1701,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8230218887329102,
-      "rewards/margins": 2.0800347328186035,
-      "rewards/rejected": -0.25701290369033813,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -1.9018220901489258,
-      "logits/rejected": -1.9121367931365967,
-      "logps/chosen": -28.98052406311035,
-      "logps/rejected": -30.033344268798828,
-      "loss": 0.184,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.628839135169983,
-      "rewards/margins": 1.8717044591903687,
-      "rewards/rejected": -0.24286529421806335,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.8325939178466797,
-      "logits/rejected": -1.8257277011871338,
-      "logps/chosen": -31.01336669921875,
-      "logps/rejected": -32.70115280151367,
-      "loss": 0.188,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.5858715772628784,
-      "rewards/margins": 1.8802779912948608,
-      "rewards/rejected": -0.29440635442733765,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.8861255645751953,
-      "logits/rejected": -1.8828990459442139,
-      "logps/chosen": -30.904102325439453,
-      "logps/rejected": -33.84788513183594,
-      "loss": 0.1889,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.6328811645507812,
-      "rewards/margins": 1.9854776859283447,
-      "rewards/rejected": -0.3525964617729187,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.1516811847686768,
-      "eval_logits/rejected": -2.146984577178955,
-      "eval_logps/chosen": -33.62123107910156,
-      "eval_logps/rejected": -37.318565368652344,
-      "eval_loss": 0.4726825952529907,
-      "eval_rewards/accuracies": 0.6004983186721802,
-      "eval_rewards/chosen": 0.24799373745918274,
-      "eval_rewards/margins": 0.12916025519371033,
-      "eval_rewards/rejected": 0.11883345246315002,
-      "eval_runtime": 145.8433,
-      "eval_samples_per_second": 2.352,
-      "eval_steps_per_second": 0.295,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.9646100997924805,
-      "logits/rejected": -1.9652503728866577,
-      "logps/chosen": -27.146465301513672,
-      "logps/rejected": -33.218292236328125,
-      "loss": 0.2041,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.6018869876861572,
-      "rewards/margins": 1.8885917663574219,
-      "rewards/rejected": -0.2867048680782318,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.8171708583831787,
-      "logits/rejected": -1.8171415328979492,
-      "logps/chosen": -30.371408462524414,
-      "logps/rejected": -34.34622573852539,
-      "loss": 0.1949,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.6022796630859375,
-      "rewards/margins": 1.885776162147522,
-      "rewards/rejected": -0.2834966778755188,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -1.9699510335922241,
-      "logits/rejected": -1.9673391580581665,
-      "logps/chosen": -28.992298126220703,
-      "logps/rejected": -32.012474060058594,
-      "loss": 0.2043,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.489058256149292,
-      "rewards/margins": 1.806349754333496,
-      "rewards/rejected": -0.3172914385795593,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.95291268825531,
-      "logits/rejected": -1.9540326595306396,
-      "logps/chosen": -28.15656089782715,
-      "logps/rejected": -30.210216522216797,
-      "loss": 0.204,
-      "rewards/accuracies": 0.970833420753479,
-      "rewards/chosen": 1.4780277013778687,
-      "rewards/margins": 1.730049729347229,
-      "rewards/rejected": -0.2520219683647156,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.19402528018146367,
-      "train_runtime": 10806.8766,
-      "train_samples_per_second": 1.14,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.5965218513042896,
+      "train_runtime": 3252.2803,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,