diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2537 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8667426109313965,
-      "logits/rejected": -1.8710602521896362,
-      "logps/chosen": -36.991912841796875,
-      "logps/rejected": -33.67206954956055,
-      "loss": 0.9773,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.0058750128373503685,
-      "rewards/margins": 0.022671451792120934,
-      "rewards/rejected": -0.01679643802344799,
+      "logits/chosen": -1.8668746948242188,
+      "logits/rejected": -1.8712046146392822,
+      "logps/chosen": -36.99528884887695,
+      "logps/rejected": -33.6615104675293,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004523592535406351,
+      "rewards/margins": 0.017096158117055893,
+      "rewards/rejected": -0.012572565115988255,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9977442026138306,
-      "logits/rejected": -2.0003952980041504,
-      "logps/chosen": -29.659366607666016,
-      "logps/rejected": -29.05437660217285,
-      "loss": 1.0105,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.006868218071758747,
-      "rewards/margins": -0.010493903420865536,
-      "rewards/rejected": 0.003625686513260007,
+      "logits/chosen": -1.9979562759399414,
+      "logits/rejected": -2.000598907470703,
+      "logps/chosen": -29.644357681274414,
+      "logps/rejected": -29.06288719177246,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0008665517088957131,
+      "rewards/margins": -0.0010885533411055803,
+      "rewards/rejected": 0.00022200122475624084,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.920693039894104,
-      "logits/rejected": -1.91802179813385,
-      "logps/chosen": -31.39971351623535,
-      "logps/rejected": -33.21495819091797,
-      "loss": 0.9948,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.006551843136548996,
-      "rewards/margins": 0.0051523735746741295,
-      "rewards/rejected": 0.001399471191689372,
+      "logits/chosen": -1.9213365316390991,
+      "logits/rejected": -1.9186455011367798,
+      "logps/chosen": -31.3991756439209,
+      "logps/rejected": -33.220787048339844,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006767785642296076,
+      "rewards/margins": 0.007698210421949625,
+      "rewards/rejected": -0.0009304238483309746,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.018057107925415,
-      "logits/rejected": -2.0093047618865967,
-      "logps/chosen": -32.565284729003906,
-      "logps/rejected": -32.50053405761719,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0046076299622654915,
-      "rewards/margins": -0.00024683662923052907,
-      "rewards/rejected": 0.004854466766119003,
+      "logits/chosen": -2.0176024436950684,
+      "logits/rejected": -2.008852481842041,
+      "logps/chosen": -32.565155029296875,
+      "logps/rejected": -32.51045608520508,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004661071114242077,
+      "rewards/margins": 0.0037749619223177433,
+      "rewards/rejected": 0.0008861090755090117,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8627817630767822,
-      "logits/rejected": -1.851999044418335,
-      "logps/chosen": -33.549964904785156,
-      "logps/rejected": -35.44340896606445,
-      "loss": 1.0006,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0030060340650379658,
-      "rewards/margins": -0.0005688609671778977,
-      "rewards/rejected": 0.003574896603822708,
+      "logits/chosen": -1.8624731302261353,
+      "logits/rejected": -1.8516931533813477,
+      "logps/chosen": -33.547359466552734,
+      "logps/rejected": -35.463809967041016,
+      "loss": 0.69,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004046988673508167,
+      "rewards/margins": 0.008634108118712902,
+      "rewards/rejected": -0.004587120376527309,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9416770935058594,
-      "logits/rejected": -1.9436094760894775,
-      "logps/chosen": -32.53351593017578,
-      "logps/rejected": -33.217529296875,
-      "loss": 0.9561,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.026666466146707535,
-      "rewards/margins": 0.04512657970190048,
-      "rewards/rejected": -0.018460111692547798,
+      "logits/chosen": -1.9406204223632812,
+      "logits/rejected": -1.9425837993621826,
+      "logps/chosen": -32.54151153564453,
+      "logps/rejected": -33.21025848388672,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.023469632491469383,
+      "rewards/margins": 0.03902136906981468,
+      "rewards/rejected": -0.015551735647022724,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.072779655456543,
-      "logits/rejected": -2.077756881713867,
-      "logps/chosen": -34.002342224121094,
-      "logps/rejected": -36.633216857910156,
-      "loss": 0.9798,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0055419160053133965,
-      "rewards/margins": 0.020196745172142982,
-      "rewards/rejected": -0.025738662108778954,
+      "logits/chosen": -2.0720112323760986,
+      "logits/rejected": -2.0769832134246826,
+      "logps/chosen": -33.98130416870117,
+      "logps/rejected": -36.64153289794922,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0028717622626572847,
+      "rewards/margins": 0.03193504735827446,
+      "rewards/rejected": -0.029063284397125244,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9332294464111328,
-      "logits/rejected": -1.9363486766815186,
-      "logps/chosen": -34.33064651489258,
-      "logps/rejected": -34.64745330810547,
-      "loss": 0.9391,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.03886651247739792,
-      "rewards/margins": 0.06088464334607124,
-      "rewards/rejected": -0.022018127143383026,
+      "logits/chosen": -1.9326984882354736,
+      "logits/rejected": -1.935831069946289,
+      "logps/chosen": -34.31356430053711,
+      "logps/rejected": -34.65351867675781,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.045696478337049484,
+      "rewards/margins": 0.07014231383800507,
+      "rewards/rejected": -0.02444584295153618,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9409536123275757,
-      "logits/rejected": -1.9454677104949951,
-      "logps/chosen": -32.381752014160156,
-      "logps/rejected": -32.35805892944336,
-      "loss": 0.9661,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0342683270573616,
-      "rewards/margins": 0.033929694443941116,
-      "rewards/rejected": 0.0003386303724255413,
+      "logits/chosen": -1.9407522678375244,
+      "logits/rejected": -1.9452556371688843,
+      "logps/chosen": -32.40108108520508,
+      "logps/rejected": -32.342872619628906,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.02653699554502964,
+      "rewards/margins": 0.020122777670621872,
+      "rewards/rejected": 0.0064142136834561825,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0382819175720215,
-      "logits/rejected": -2.036304473876953,
-      "logps/chosen": -32.15763473510742,
-      "logps/rejected": -31.302764892578125,
-      "loss": 0.9521,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.033279187977313995,
-      "rewards/margins": 0.047900475561618805,
-      "rewards/rejected": -0.014621290378272533,
+      "logits/chosen": -2.0378258228302,
+      "logits/rejected": -2.0358424186706543,
+      "logps/chosen": -32.15534973144531,
+      "logps/rejected": -31.297805786132812,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03419475629925728,
+      "rewards/margins": 0.04683210328221321,
+      "rewards/rejected": -0.01263735257089138,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2327499389648438,
-      "eval_logits/rejected": -2.227902889251709,
-      "eval_logps/chosen": -34.038063049316406,
-      "eval_logps/rejected": -37.54012680053711,
-      "eval_loss": 0.9920137524604797,
-      "eval_rewards/accuracies": 0.5361295938491821,
-      "eval_rewards/chosen": -0.0014053798513486981,
-      "eval_rewards/margins": 0.007998107932507992,
-      "eval_rewards/rejected": -0.009403487667441368,
-      "eval_runtime": 146.1087,
-      "eval_samples_per_second": 2.348,
+      "eval_logits/chosen": -2.233442544937134,
+      "eval_logits/rejected": -2.2285873889923096,
+      "eval_logps/chosen": -34.044193267822266,
+      "eval_logps/rejected": -37.55242919921875,
+      "eval_loss": 0.6902939081192017,
+      "eval_rewards/accuracies": 0.5481727719306946,
+      "eval_rewards/chosen": -0.0038560593966394663,
+      "eval_rewards/margins": 0.010466905310750008,
+      "eval_rewards/rejected": -0.014322965405881405,
+      "eval_runtime": 146.038,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9931761026382446,
-      "logits/rejected": -1.9907801151275635,
-      "logps/chosen": -33.120845794677734,
-      "logps/rejected": -34.02234649658203,
-      "loss": 0.9536,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.04918808490037918,
-      "rewards/margins": 0.046377379447221756,
-      "rewards/rejected": 0.0028106991667300463,
+      "logits/chosen": -1.9928157329559326,
+      "logits/rejected": -1.9904266595840454,
+      "logps/chosen": -33.1363410949707,
+      "logps/rejected": -34.00283432006836,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04299246892333031,
+      "rewards/margins": 0.03237856179475784,
+      "rewards/rejected": 0.010613908991217613,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.004035472869873,
-      "logits/rejected": -1.9957201480865479,
-      "logps/chosen": -32.31081771850586,
-      "logps/rejected": -32.129127502441406,
-      "loss": 0.9593,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.05363499000668526,
-      "rewards/margins": 0.04068244248628616,
-      "rewards/rejected": 0.012952548451721668,
+      "logits/chosen": -2.0046679973602295,
+      "logits/rejected": -1.9963252544403076,
+      "logps/chosen": -32.3227653503418,
+      "logps/rejected": -32.157779693603516,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04885613173246384,
+      "rewards/margins": 0.0473661907017231,
+      "rewards/rejected": 0.0014899425441399217,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0312001705169678,
-      "logits/rejected": -2.023239850997925,
-      "logps/chosen": -30.317270278930664,
-      "logps/rejected": -32.05809783935547,
-      "loss": 0.9384,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06337819993495941,
-      "rewards/margins": 0.06541591882705688,
-      "rewards/rejected": -0.0020377314649522305,
+      "logits/chosen": -2.0337133407592773,
+      "logits/rejected": -2.0257389545440674,
+      "logps/chosen": -30.330951690673828,
+      "logps/rejected": -32.077327728271484,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.05790700390934944,
+      "rewards/margins": 0.06763499230146408,
+      "rewards/rejected": -0.00972799677401781,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9611848592758179,
-      "logits/rejected": -1.9714059829711914,
-      "logps/chosen": -31.235523223876953,
-      "logps/rejected": -32.55936813354492,
-      "loss": 0.9165,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.07365532219409943,
-      "rewards/margins": 0.08345074951648712,
-      "rewards/rejected": -0.009795431979000568,
+      "logits/chosen": -1.9636684656143188,
+      "logits/rejected": -1.9739125967025757,
+      "logps/chosen": -31.2012996673584,
+      "logps/rejected": -32.56267547607422,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.08734508603811264,
+      "rewards/margins": 0.09846383333206177,
+      "rewards/rejected": -0.011118754744529724,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8718490600585938,
-      "logits/rejected": -1.8730093240737915,
-      "logps/chosen": -33.873046875,
-      "logps/rejected": -34.80980682373047,
-      "loss": 0.8563,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12785716354846954,
-      "rewards/margins": 0.1488770991563797,
-      "rewards/rejected": -0.021019931882619858,
+      "logits/chosen": -1.8753896951675415,
+      "logits/rejected": -1.8765614032745361,
+      "logps/chosen": -33.930477142333984,
+      "logps/rejected": -34.81265640258789,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10488543659448624,
+      "rewards/margins": 0.1270444542169571,
+      "rewards/rejected": -0.022159017622470856,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9235217571258545,
-      "logits/rejected": -1.9201278686523438,
-      "logps/chosen": -36.01029968261719,
-      "logps/rejected": -32.690834045410156,
-      "loss": 0.9482,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.06455211341381073,
-      "rewards/margins": 0.05178683251142502,
-      "rewards/rejected": 0.012765283696353436,
+      "logits/chosen": -1.9266326427459717,
+      "logits/rejected": -1.9231981039047241,
+      "logps/chosen": -36.001312255859375,
+      "logps/rejected": -32.70484924316406,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0681454986333847,
+      "rewards/margins": 0.06098458915948868,
+      "rewards/rejected": 0.007160906679928303,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0244345664978027,
-      "logits/rejected": -2.0171027183532715,
-      "logps/chosen": -33.474937438964844,
-      "logps/rejected": -31.42409324645996,
-      "loss": 0.8419,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.12916772067546844,
-      "rewards/margins": 0.15806543827056885,
-      "rewards/rejected": -0.028897713869810104,
+      "logits/chosen": -2.0281758308410645,
+      "logits/rejected": -2.0208277702331543,
+      "logps/chosen": -33.47995376586914,
+      "logps/rejected": -31.4173526763916,
+      "loss": 0.6278,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.12716376781463623,
+      "rewards/margins": 0.15336360037326813,
+      "rewards/rejected": -0.026199836283922195,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0310873985290527,
-      "logits/rejected": -2.036334276199341,
-      "logps/chosen": -32.2062873840332,
-      "logps/rejected": -32.42302703857422,
-      "loss": 0.8919,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1354007124900818,
-      "rewards/margins": 0.11055666208267212,
-      "rewards/rejected": 0.024844054132699966,
+      "logits/chosen": -2.033982038497925,
+      "logits/rejected": -2.0392203330993652,
+      "logps/chosen": -32.239437103271484,
+      "logps/rejected": -32.44209671020508,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12213903665542603,
+      "rewards/margins": 0.10492189973592758,
+      "rewards/rejected": 0.017217133194208145,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0314042568206787,
-      "logits/rejected": -2.02862286567688,
-      "logps/chosen": -31.278457641601562,
-      "logps/rejected": -31.348251342773438,
-      "loss": 0.9038,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0841989517211914,
-      "rewards/margins": 0.09621445834636688,
-      "rewards/rejected": -0.012015508487820625,
+      "logits/chosen": -2.034848928451538,
+      "logits/rejected": -2.032095432281494,
+      "logps/chosen": -31.259105682373047,
+      "logps/rejected": -31.31390953063965,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09193893522024155,
+      "rewards/margins": 0.09021683037281036,
+      "rewards/rejected": 0.0017221048474311829,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9020229578018188,
-      "logits/rejected": -1.906660795211792,
-      "logps/chosen": -31.28672218322754,
-      "logps/rejected": -32.84270477294922,
-      "loss": 0.8515,
+      "logits/chosen": -1.9051767587661743,
+      "logits/rejected": -1.9098323583602905,
+      "logps/chosen": -31.32687759399414,
+      "logps/rejected": -32.82324981689453,
+      "loss": 0.6419,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12782469391822815,
-      "rewards/margins": 0.14853176474571228,
-      "rewards/rejected": -0.020707078278064728,
+      "rewards/chosen": 0.1117621660232544,
+      "rewards/margins": 0.12468986213207245,
+      "rewards/rejected": -0.012927706353366375,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.22743558883667,
-      "eval_logits/rejected": -2.2226204872131348,
-      "eval_logps/chosen": -34.04505920410156,
-      "eval_logps/rejected": -37.573036193847656,
-      "eval_loss": 0.9816663265228271,
-      "eval_rewards/accuracies": 0.559385359287262,
-      "eval_rewards/chosen": -0.004204160068184137,
-      "eval_rewards/margins": 0.018361244350671768,
-      "eval_rewards/rejected": -0.022565403953194618,
-      "eval_runtime": 145.491,
-      "eval_samples_per_second": 2.358,
-      "eval_steps_per_second": 0.296,
+      "eval_logits/chosen": -2.2309587001800537,
+      "eval_logits/rejected": -2.2261300086975098,
+      "eval_logps/chosen": -34.06596374511719,
+      "eval_logps/rejected": -37.5656852722168,
+      "eval_loss": 0.6932108402252197,
+      "eval_rewards/accuracies": 0.5041528344154358,
+      "eval_rewards/chosen": -0.0125651890411973,
+      "eval_rewards/margins": 0.007060302421450615,
+      "eval_rewards/rejected": -0.01962549053132534,
+      "eval_runtime": 145.8797,
+      "eval_samples_per_second": 2.351,
+      "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0146515369415283,
-      "logits/rejected": -2.025285005569458,
-      "logps/chosen": -31.747217178344727,
-      "logps/rejected": -33.96654510498047,
-      "loss": 0.8755,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.08963600546121597,
-      "rewards/margins": 0.13117524981498718,
-      "rewards/rejected": -0.04153924435377121,
+      "logits/chosen": -2.0179035663604736,
+      "logits/rejected": -2.0285422801971436,
+      "logps/chosen": -31.756057739257812,
+      "logps/rejected": -33.936058044433594,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08609838038682938,
+      "rewards/margins": 0.11544252932071686,
+      "rewards/rejected": -0.029344135895371437,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9072997570037842,
-      "logits/rejected": -1.922041654586792,
-      "logps/chosen": -29.81575584411621,
-      "logps/rejected": -31.636306762695312,
-      "loss": 0.8509,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11831430345773697,
-      "rewards/margins": 0.14956556260585785,
-      "rewards/rejected": -0.03125125169754028,
+      "logits/chosen": -1.910278081893921,
+      "logits/rejected": -1.9250171184539795,
+      "logps/chosen": -29.805334091186523,
+      "logps/rejected": -31.600574493408203,
+      "loss": 0.6326,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1224818006157875,
+      "rewards/margins": 0.139441579580307,
+      "rewards/rejected": -0.016959769651293755,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9637253284454346,
-      "logits/rejected": -1.9677000045776367,
-      "logps/chosen": -33.084877014160156,
-      "logps/rejected": -31.639108657836914,
-      "loss": 0.8476,
+      "logits/chosen": -1.9668325185775757,
+      "logits/rejected": -1.970798134803772,
+      "logps/chosen": -33.078094482421875,
+      "logps/rejected": -31.6514949798584,
+      "loss": 0.621,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13303686678409576,
-      "rewards/margins": 0.16895917057991028,
-      "rewards/rejected": -0.03592229634523392,
+      "rewards/chosen": 0.13575060665607452,
+      "rewards/margins": 0.1766270101070404,
+      "rewards/rejected": -0.04087639972567558,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9612106084823608,
-      "logits/rejected": -1.9394094944000244,
-      "logps/chosen": -33.84415817260742,
-      "logps/rejected": -35.132205963134766,
-      "loss": 0.8154,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1238357201218605,
-      "rewards/margins": 0.19639968872070312,
-      "rewards/rejected": -0.07256398350000381,
+      "logits/chosen": -1.965409278869629,
+      "logits/rejected": -1.9435851573944092,
+      "logps/chosen": -33.83237075805664,
+      "logps/rejected": -35.09648132324219,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12855152785778046,
+      "rewards/margins": 0.18682697415351868,
+      "rewards/rejected": -0.05827543884515762,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.002776861190796,
-      "logits/rejected": -1.999464988708496,
-      "logps/chosen": -32.74934387207031,
-      "logps/rejected": -36.28064727783203,
-      "loss": 0.9096,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0702909380197525,
-      "rewards/margins": 0.09257940202951431,
-      "rewards/rejected": -0.02228846587240696,
+      "logits/chosen": -2.006166696548462,
+      "logits/rejected": -2.002840042114258,
+      "logps/chosen": -32.69712448120117,
+      "logps/rejected": -36.255043029785156,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09117679297924042,
+      "rewards/margins": 0.10322580486536026,
+      "rewards/rejected": -0.01204901933670044,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8692924976348877,
-      "logits/rejected": -1.8668625354766846,
-      "logps/chosen": -33.982818603515625,
-      "logps/rejected": -35.535736083984375,
-      "loss": 0.9053,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.08223171532154083,
-      "rewards/margins": 0.09467832744121552,
-      "rewards/rejected": -0.012446624226868153,
+      "logits/chosen": -1.8737766742706299,
+      "logits/rejected": -1.87137770652771,
+      "logps/chosen": -33.97405242919922,
+      "logps/rejected": -35.56011962890625,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.08574129641056061,
+      "rewards/margins": 0.1079399362206459,
+      "rewards/rejected": -0.022198637947440147,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8543907403945923,
-      "logits/rejected": -1.8520078659057617,
-      "logps/chosen": -34.20638656616211,
-      "logps/rejected": -31.873388290405273,
-      "loss": 0.8923,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.07648433744907379,
-      "rewards/margins": 0.11730633676052094,
-      "rewards/rejected": -0.040821999311447144,
+      "logits/chosen": -1.8586937189102173,
+      "logits/rejected": -1.8563038110733032,
+      "logps/chosen": -34.204627990722656,
+      "logps/rejected": -31.85955810546875,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.07718975841999054,
+      "rewards/margins": 0.11248154938220978,
+      "rewards/rejected": -0.03529178351163864,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9576927423477173,
-      "logits/rejected": -1.9472328424453735,
-      "logps/chosen": -34.999000549316406,
-      "logps/rejected": -31.88382911682129,
-      "loss": 0.8398,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.1430555135011673,
-      "rewards/margins": 0.16019006073474884,
-      "rewards/rejected": -0.017134560272097588,
+      "logits/chosen": -1.962044358253479,
+      "logits/rejected": -1.9515289068222046,
+      "logps/chosen": -35.0092887878418,
+      "logps/rejected": -31.877635955810547,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.13894058763980865,
+      "rewards/margins": 0.15359947085380554,
+      "rewards/rejected": -0.01465887576341629,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0523831844329834,
-      "logits/rejected": -2.037504196166992,
-      "logps/chosen": -30.712234497070312,
-      "logps/rejected": -32.659637451171875,
-      "loss": 0.9146,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.08475608378648758,
-      "rewards/margins": 0.08787757158279419,
-      "rewards/rejected": -0.0031214915215969086,
+      "logits/chosen": -2.0572900772094727,
+      "logits/rejected": -2.0424036979675293,
+      "logps/chosen": -30.72947120666504,
+      "logps/rejected": -32.645362854003906,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.07786226272583008,
+      "rewards/margins": 0.07527298480272293,
+      "rewards/rejected": 0.002589278621599078,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9236303567886353,
-      "logits/rejected": -1.9211324453353882,
-      "logps/chosen": -32.370330810546875,
-      "logps/rejected": -30.952068328857422,
-      "loss": 0.7473,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.2227112501859665,
-      "rewards/margins": 0.27681466937065125,
-      "rewards/rejected": -0.054103411734104156,
+      "logits/chosen": -1.9281587600708008,
+      "logits/rejected": -1.9256232976913452,
+      "logps/chosen": -32.37285614013672,
+      "logps/rejected": -30.912500381469727,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.22170230746269226,
+      "rewards/margins": 0.2599778473377228,
+      "rewards/rejected": -0.03827553242444992,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.223449468612671,
-      "eval_logits/rejected": -2.2186241149902344,
-      "eval_logps/chosen": -34.09219741821289,
-      "eval_logps/rejected": -37.61648178100586,
-      "eval_loss": 0.9830461740493774,
-      "eval_rewards/accuracies": 0.5390365719795227,
-      "eval_rewards/chosen": -0.023058738559484482,
-      "eval_rewards/margins": 0.016884688287973404,
-      "eval_rewards/rejected": -0.03994342312216759,
-      "eval_runtime": 145.6848,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2279410362243652,
+      "eval_logits/rejected": -2.2231125831604004,
+      "eval_logps/chosen": -34.09661865234375,
+      "eval_logps/rejected": -37.598716735839844,
+      "eval_loss": 0.6926039457321167,
+      "eval_rewards/accuracies": 0.49543190002441406,
+      "eval_rewards/chosen": -0.024825766682624817,
+      "eval_rewards/margins": 0.008013932965695858,
+      "eval_rewards/rejected": -0.03283970057964325,
+      "eval_runtime": 145.6649,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 19.375,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -2.057250499725342,
-      "logits/rejected": -2.0444836616516113,
-      "logps/chosen": -32.08330535888672,
-      "logps/rejected": -32.93002700805664,
-      "loss": 0.7174,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.17803600430488586,
-      "rewards/margins": 0.29362356662750244,
-      "rewards/rejected": -0.11558754742145538,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.9122984409332275,
+      "logits/rejected": -1.9090602397918701,
+      "logps/chosen": -31.30501365661621,
+      "logps/rejected": -33.817237854003906,
+      "loss": 0.6314,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11933410167694092,
+      "rewards/margins": 0.149368017911911,
+      "rewards/rejected": -0.030033910647034645,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 18.375,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9667476415634155,
-      "logits/rejected": -1.9581434726715088,
-      "logps/chosen": -31.69952964782715,
-      "logps/rejected": -30.477031707763672,
-      "loss": 0.6654,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.26218193769454956,
-      "rewards/margins": 0.35643792152404785,
-      "rewards/rejected": -0.09425600618124008,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.962451696395874,
+      "logits/rejected": -1.950260877609253,
+      "logps/chosen": -34.321712493896484,
+      "logps/rejected": -33.68184280395508,
+      "loss": 0.6222,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.10974420607089996,
+      "rewards/margins": 0.16694548726081848,
+      "rewards/rejected": -0.0572013333439827,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 13.5,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.902342438697815,
-      "logits/rejected": -1.914393424987793,
-      "logps/chosen": -29.752674102783203,
-      "logps/rejected": -33.74560546875,
-      "loss": 0.5789,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.29063910245895386,
-      "rewards/margins": 0.427977979183197,
-      "rewards/rejected": -0.13733889162540436,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -1.9975078105926514,
+      "logits/rejected": -1.996080994606018,
+      "logps/chosen": -33.17241287231445,
+      "logps/rejected": -32.53407669067383,
+      "loss": 0.6313,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12535209953784943,
+      "rewards/margins": 0.14715085923671722,
+      "rewards/rejected": -0.021798742935061455,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 23.375,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -1.8653125762939453,
-      "logits/rejected": -1.8561639785766602,
-      "logps/chosen": -33.982444763183594,
-      "logps/rejected": -36.24190902709961,
-      "loss": 0.6225,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2865905463695526,
-      "rewards/margins": 0.44066372513771057,
-      "rewards/rejected": -0.15407314896583557,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.0841541290283203,
+      "logits/rejected": -2.0684666633605957,
+      "logps/chosen": -33.79545211791992,
+      "logps/rejected": -33.1196174621582,
+      "loss": 0.6276,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15357479453086853,
+      "rewards/margins": 0.15420618653297424,
+      "rewards/rejected": -0.000631416798569262,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 16.875,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.9161920547485352,
-      "logits/rejected": -1.9168342351913452,
-      "logps/chosen": -33.598670959472656,
-      "logps/rejected": -34.16885757446289,
-      "loss": 0.6015,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.3226447105407715,
-      "rewards/margins": 0.4505980908870697,
-      "rewards/rejected": -0.1279533952474594,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9566657543182373,
+      "logits/rejected": -1.9558334350585938,
+      "logps/chosen": -32.842193603515625,
+      "logps/rejected": -32.53485870361328,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.17596027255058289,
+      "rewards/margins": 0.2038455456495285,
+      "rewards/rejected": -0.027885273098945618,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 17.75,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -2.0425257682800293,
-      "logits/rejected": -2.0486459732055664,
-      "logps/chosen": -31.08384132385254,
-      "logps/rejected": -33.032264709472656,
-      "loss": 0.6861,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.19576172530651093,
-      "rewards/margins": 0.3413549065589905,
-      "rewards/rejected": -0.14559319615364075,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9120171070098877,
+      "logits/rejected": -1.9222869873046875,
+      "logps/chosen": -31.887542724609375,
+      "logps/rejected": -35.35187911987305,
+      "loss": 0.6293,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.12685921788215637,
+      "rewards/margins": 0.15210750699043274,
+      "rewards/rejected": -0.025248277932405472,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 20.5,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -1.9708837270736694,
-      "logits/rejected": -1.970446228981018,
-      "logps/chosen": -32.829166412353516,
-      "logps/rejected": -36.46767044067383,
-      "loss": 0.6072,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.24631083011627197,
-      "rewards/margins": 0.41346845030784607,
-      "rewards/rejected": -0.16715766489505768,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.0516254901885986,
+      "logits/rejected": -2.045139789581299,
+      "logps/chosen": -33.33611297607422,
+      "logps/rejected": -29.27042579650879,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.12634168565273285,
+      "rewards/margins": 0.14394234120845795,
+      "rewards/rejected": -0.01760067418217659,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 19.0,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -2.002481698989868,
-      "logits/rejected": -1.9979221820831299,
-      "logps/chosen": -30.1049747467041,
-      "logps/rejected": -29.63289451599121,
-      "loss": 0.6387,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.2422492504119873,
-      "rewards/margins": 0.40985146164894104,
-      "rewards/rejected": -0.16760216653347015,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9114952087402344,
+      "logits/rejected": -1.9137216806411743,
+      "logps/chosen": -33.884849548339844,
+      "logps/rejected": -30.985692977905273,
+      "loss": 0.6144,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.14167849719524384,
+      "rewards/margins": 0.18690630793571472,
+      "rewards/rejected": -0.04522782564163208,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "grad_norm": 22.625,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.773651123046875,
-      "logits/rejected": -1.7800319194793701,
-      "logps/chosen": -31.610965728759766,
-      "logps/rejected": -36.748817443847656,
-      "loss": 0.523,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.30989953875541687,
-      "rewards/margins": 0.5621098279953003,
-      "rewards/rejected": -0.25221019983291626,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "grad_norm": 17.75,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.9248863458633423,
-      "logits/rejected": -1.918617844581604,
-      "logps/chosen": -32.99136734008789,
-      "logps/rejected": -32.704185485839844,
-      "loss": 0.5492,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.33653250336647034,
-      "rewards/margins": 0.49465543031692505,
-      "rewards/rejected": -0.15812288224697113,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.1900622844696045,
-      "eval_logits/rejected": -2.185243606567383,
-      "eval_logps/chosen": -34.19782638549805,
-      "eval_logps/rejected": -37.779701232910156,
-      "eval_loss": 0.9599968194961548,
-      "eval_rewards/accuracies": 0.5564784407615662,
-      "eval_rewards/chosen": -0.06531065702438354,
-      "eval_rewards/margins": 0.03992270678281784,
-      "eval_rewards/rejected": -0.10523335635662079,
-      "eval_runtime": 145.4889,
-      "eval_samples_per_second": 2.358,
-      "eval_steps_per_second": 0.296,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "grad_norm": 14.9375,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -1.9203557968139648,
-      "logits/rejected": -1.9276371002197266,
-      "logps/chosen": -28.529979705810547,
-      "logps/rejected": -29.745304107666016,
-      "loss": 0.5725,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.23615682125091553,
-      "rewards/margins": 0.4949887692928314,
-      "rewards/rejected": -0.2588319778442383,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "grad_norm": 17.0,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.9383413791656494,
-      "logits/rejected": -1.9373687505722046,
-      "logps/chosen": -33.60287094116211,
-      "logps/rejected": -31.25448989868164,
-      "loss": 0.6687,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2591347098350525,
-      "rewards/margins": 0.3837122917175293,
-      "rewards/rejected": -0.12457761913537979,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "grad_norm": 16.875,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.9323232173919678,
-      "logits/rejected": -1.9150965213775635,
-      "logps/chosen": -30.429235458374023,
-      "logps/rejected": -33.68767547607422,
-      "loss": 0.5576,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.21918778121471405,
-      "rewards/margins": 0.5237432718276978,
-      "rewards/rejected": -0.3045555651187897,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "grad_norm": 17.5,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -1.9633067846298218,
-      "logits/rejected": -1.9654438495635986,
-      "logps/chosen": -35.656089782714844,
-      "logps/rejected": -35.58705520629883,
-      "loss": 0.4182,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3825095295906067,
-      "rewards/margins": 0.6733442544937134,
-      "rewards/rejected": -0.2908347249031067,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "grad_norm": 16.75,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -2.039092779159546,
-      "logits/rejected": -2.03885817527771,
-      "logps/chosen": -31.8747501373291,
-      "logps/rejected": -34.06696319580078,
-      "loss": 0.5685,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3034403622150421,
-      "rewards/margins": 0.5061808228492737,
-      "rewards/rejected": -0.2027404010295868,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "grad_norm": 18.0,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -1.9820168018341064,
-      "logits/rejected": -1.974509835243225,
-      "logps/chosen": -31.496234893798828,
-      "logps/rejected": -36.78831100463867,
-      "loss": 0.435,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3638893961906433,
-      "rewards/margins": 0.6575988531112671,
-      "rewards/rejected": -0.2937094271183014,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "grad_norm": 14.75,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.8861879110336304,
-      "logits/rejected": -1.8827335834503174,
-      "logps/chosen": -33.01527404785156,
-      "logps/rejected": -33.59056854248047,
-      "loss": 0.4878,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3668093681335449,
-      "rewards/margins": 0.6334856748580933,
-      "rewards/rejected": -0.26667624711990356,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 14.625,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -2.0107829570770264,
-      "logits/rejected": -2.003845691680908,
-      "logps/chosen": -30.23215103149414,
-      "logps/rejected": -32.65163803100586,
-      "loss": 0.5881,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.2385544776916504,
-      "rewards/margins": 0.5018148422241211,
-      "rewards/rejected": -0.2632603049278259,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "grad_norm": 11.625,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9544703960418701,
-      "logits/rejected": -1.9560562372207642,
-      "logps/chosen": -34.39679718017578,
-      "logps/rejected": -34.34023666381836,
-      "loss": 0.4545,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.38255801796913147,
-      "rewards/margins": 0.6620479822158813,
-      "rewards/rejected": -0.2794899046421051,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 16.75,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.8426096439361572,
-      "logits/rejected": -1.840172529220581,
-      "logps/chosen": -32.309085845947266,
-      "logps/rejected": -37.37702178955078,
-      "loss": 0.4504,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.345283180475235,
-      "rewards/margins": 0.7103637456893921,
-      "rewards/rejected": -0.3650805950164795,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.1818087100982666,
-      "eval_logits/rejected": -2.1769938468933105,
-      "eval_logps/chosen": -34.38530349731445,
-      "eval_logps/rejected": -37.95327377319336,
-      "eval_loss": 0.9655585289001465,
-      "eval_rewards/accuracies": 0.5685215592384338,
-      "eval_rewards/chosen": -0.14030161499977112,
-      "eval_rewards/margins": 0.03436039760708809,
-      "eval_rewards/rejected": -0.1746620088815689,
-      "eval_runtime": 145.206,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "grad_norm": 13.4375,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.9420890808105469,
-      "logits/rejected": -1.9468368291854858,
-      "logps/chosen": -32.48661804199219,
-      "logps/rejected": -32.22886276245117,
-      "loss": 0.4787,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.38829392194747925,
-      "rewards/margins": 0.6213949918746948,
-      "rewards/rejected": -0.23310112953186035,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "grad_norm": 14.8125,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -1.9701076745986938,
-      "logits/rejected": -1.981092095375061,
-      "logps/chosen": -30.474559783935547,
-      "logps/rejected": -32.24431610107422,
-      "loss": 0.4762,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.36634185910224915,
-      "rewards/margins": 0.6609948873519897,
-      "rewards/rejected": -0.2946530282497406,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "grad_norm": 10.4375,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.897570252418518,
-      "logits/rejected": -1.8943248987197876,
-      "logps/chosen": -31.66860580444336,
-      "logps/rejected": -33.14496994018555,
-      "loss": 0.4876,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.30076172947883606,
-      "rewards/margins": 0.6136397123336792,
-      "rewards/rejected": -0.3128780126571655,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 14.0,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.7630268335342407,
-      "logits/rejected": -1.7722015380859375,
-      "logps/chosen": -31.767126083374023,
-      "logps/rejected": -32.539676666259766,
-      "loss": 0.5367,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3209052085876465,
-      "rewards/margins": 0.5943812727928162,
-      "rewards/rejected": -0.27347609400749207,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "grad_norm": 13.9375,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.856566071510315,
-      "logits/rejected": -1.8504016399383545,
-      "logps/chosen": -32.787315368652344,
-      "logps/rejected": -31.569156646728516,
-      "loss": 0.5202,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.36839646100997925,
-      "rewards/margins": 0.5995180606842041,
-      "rewards/rejected": -0.23112154006958008,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "grad_norm": 13.5625,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -1.943418264389038,
-      "logits/rejected": -1.941474199295044,
-      "logps/chosen": -33.62010192871094,
-      "logps/rejected": -32.09187316894531,
-      "loss": 0.5174,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.3804393410682678,
-      "rewards/margins": 0.6182351112365723,
-      "rewards/rejected": -0.23779578506946564,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "grad_norm": 13.9375,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.778063416481018,
-      "logits/rejected": -1.7760202884674072,
-      "logps/chosen": -31.914138793945312,
-      "logps/rejected": -31.619396209716797,
-      "loss": 0.5162,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.352909654378891,
-      "rewards/margins": 0.5997473001480103,
-      "rewards/rejected": -0.24683766067028046,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "grad_norm": 18.25,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -1.9141117334365845,
-      "logits/rejected": -1.910735845565796,
-      "logps/chosen": -30.1756591796875,
-      "logps/rejected": -35.52814865112305,
-      "loss": 0.441,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.37159496545791626,
-      "rewards/margins": 0.7323898077011108,
-      "rewards/rejected": -0.36079490184783936,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "grad_norm": 12.5,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.8503096103668213,
-      "logits/rejected": -1.8538156747817993,
-      "logps/chosen": -28.902246475219727,
-      "logps/rejected": -31.208913803100586,
-      "loss": 0.5458,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2851153612136841,
-      "rewards/margins": 0.5051326751708984,
-      "rewards/rejected": -0.22001728415489197,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "grad_norm": 14.6875,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.8863645792007446,
-      "logits/rejected": -1.8863203525543213,
-      "logps/chosen": -31.125957489013672,
-      "logps/rejected": -31.79779624938965,
-      "loss": 0.4511,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.4382299780845642,
-      "rewards/margins": 0.6801418662071228,
-      "rewards/rejected": -0.2419118881225586,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.161299228668213,
-      "eval_logits/rejected": -2.1565115451812744,
-      "eval_logps/chosen": -34.24331283569336,
-      "eval_logps/rejected": -37.89296340942383,
-      "eval_loss": 0.9337846636772156,
-      "eval_rewards/accuracies": 0.595099687576294,
-      "eval_rewards/chosen": -0.08350436389446259,
-      "eval_rewards/margins": 0.06703372299671173,
-      "eval_rewards/rejected": -0.1505381017923355,
-      "eval_runtime": 145.1637,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "grad_norm": 18.625,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -1.9492238759994507,
-      "logits/rejected": -1.950005292892456,
-      "logps/chosen": -33.86152648925781,
-      "logps/rejected": -33.876121520996094,
-      "loss": 0.4213,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.4674473702907562,
-      "rewards/margins": 0.7842208743095398,
-      "rewards/rejected": -0.3167734146118164,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "grad_norm": 16.25,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -1.9146831035614014,
-      "logits/rejected": -1.912245750427246,
-      "logps/chosen": -32.04179000854492,
-      "logps/rejected": -32.73381042480469,
-      "loss": 0.4604,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.4334433972835541,
-      "rewards/margins": 0.6990013122558594,
-      "rewards/rejected": -0.2655579149723053,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "grad_norm": 21.25,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.9271072149276733,
-      "logits/rejected": -1.9181263446807861,
-      "logps/chosen": -31.6562557220459,
-      "logps/rejected": -31.68838119506836,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.44794517755508423,
-      "rewards/margins": 0.7491516470909119,
-      "rewards/rejected": -0.30120649933815,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "grad_norm": 19.25,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.928035020828247,
-      "logits/rejected": -1.9247719049453735,
-      "logps/chosen": -31.21347427368164,
-      "logps/rejected": -33.109100341796875,
-      "loss": 0.4472,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.3939800262451172,
-      "rewards/margins": 0.669854462146759,
-      "rewards/rejected": -0.27587443590164185,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "grad_norm": 17.125,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -1.936981201171875,
-      "logits/rejected": -1.9392879009246826,
-      "logps/chosen": -32.55559158325195,
-      "logps/rejected": -34.394744873046875,
-      "loss": 0.3874,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.4392463266849518,
-      "rewards/margins": 0.7237855792045593,
-      "rewards/rejected": -0.28453919291496277,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "grad_norm": 22.75,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.848418951034546,
-      "logits/rejected": -1.8488578796386719,
-      "logps/chosen": -32.838523864746094,
-      "logps/rejected": -32.17537307739258,
-      "loss": 0.4407,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.44332486391067505,
-      "rewards/margins": 0.6705711483955383,
-      "rewards/rejected": -0.22724632918834686,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "grad_norm": 21.0,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.8874480724334717,
-      "logits/rejected": -1.8778259754180908,
-      "logps/chosen": -32.84857940673828,
-      "logps/rejected": -33.05742645263672,
-      "loss": 0.382,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.4610818326473236,
-      "rewards/margins": 0.7609356641769409,
-      "rewards/rejected": -0.29985374212265015,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "grad_norm": 19.0,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.7845916748046875,
-      "logits/rejected": -1.7813985347747803,
-      "logps/chosen": -30.31686782836914,
-      "logps/rejected": -35.026180267333984,
-      "loss": 0.3954,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.45763739943504333,
-      "rewards/margins": 0.7406758069992065,
-      "rewards/rejected": -0.2830384373664856,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "grad_norm": 14.75,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.8450958728790283,
-      "logits/rejected": -1.8443653583526611,
-      "logps/chosen": -33.978050231933594,
-      "logps/rejected": -36.23003387451172,
-      "loss": 0.3186,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.5168943405151367,
-      "rewards/margins": 0.8673771619796753,
-      "rewards/rejected": -0.35048285126686096,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "grad_norm": 11.9375,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.8192574977874756,
-      "logits/rejected": -1.822011947631836,
-      "logps/chosen": -31.353174209594727,
-      "logps/rejected": -36.575157165527344,
-      "loss": 0.3805,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.4998700022697449,
-      "rewards/margins": 0.873264491558075,
-      "rewards/rejected": -0.37339454889297485,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.1489837169647217,
-      "eval_logits/rejected": -2.1442174911499023,
-      "eval_logps/chosen": -34.29972457885742,
-      "eval_logps/rejected": -37.9394416809082,
-      "eval_loss": 0.9385247826576233,
-      "eval_rewards/accuracies": 0.5510797500610352,
-      "eval_rewards/chosen": -0.10606933385133743,
-      "eval_rewards/margins": 0.06305938214063644,
-      "eval_rewards/rejected": -0.16912870109081268,
-      "eval_runtime": 145.1991,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "grad_norm": 16.0,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -1.9265644550323486,
-      "logits/rejected": -1.9291988611221313,
-      "logps/chosen": -33.78008270263672,
-      "logps/rejected": -34.95719909667969,
-      "loss": 0.4644,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.44620281457901,
-      "rewards/margins": 0.7210227847099304,
-      "rewards/rejected": -0.2748199999332428,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "grad_norm": 20.875,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.7806246280670166,
-      "logits/rejected": -1.7747358083724976,
-      "logps/chosen": -33.649070739746094,
-      "logps/rejected": -33.15039825439453,
-      "loss": 0.4038,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.4736657738685608,
-      "rewards/margins": 0.7930890917778015,
-      "rewards/rejected": -0.3194233477115631,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 12.9375,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -1.9554284811019897,
-      "logits/rejected": -1.9523767232894897,
-      "logps/chosen": -29.604736328125,
-      "logps/rejected": -32.440574645996094,
-      "loss": 0.3749,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4192999005317688,
-      "rewards/margins": 0.8162514567375183,
-      "rewards/rejected": -0.39695149660110474,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "grad_norm": 16.125,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.7934890985488892,
-      "logits/rejected": -1.7935874462127686,
-      "logps/chosen": -32.12221145629883,
-      "logps/rejected": -38.396541595458984,
-      "loss": 0.3706,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.5363865494728088,
-      "rewards/margins": 0.8438798785209656,
-      "rewards/rejected": -0.3074934482574463,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "grad_norm": 9.125,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.6800241470336914,
-      "logits/rejected": -1.685310959815979,
-      "logps/chosen": -35.7508659362793,
-      "logps/rejected": -34.849327087402344,
-      "loss": 0.4332,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.47263655066490173,
-      "rewards/margins": 0.8234331011772156,
-      "rewards/rejected": -0.35079658031463623,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "grad_norm": 11.0,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.8796085119247437,
-      "logits/rejected": -1.8812284469604492,
-      "logps/chosen": -31.44708251953125,
-      "logps/rejected": -33.469627380371094,
-      "loss": 0.3873,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.46344003081321716,
-      "rewards/margins": 0.7396036982536316,
-      "rewards/rejected": -0.27616366744041443,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 20.25,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.8549083471298218,
-      "logits/rejected": -1.8539108037948608,
-      "logps/chosen": -30.935928344726562,
-      "logps/rejected": -35.55149841308594,
-      "loss": 0.4385,
-      "rewards/accuracies": 0.8708333969116211,
-      "rewards/chosen": 0.4958726763725281,
-      "rewards/margins": 0.7052101492881775,
-      "rewards/rejected": -0.20933744311332703,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "grad_norm": 12.8125,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.8549137115478516,
-      "logits/rejected": -1.8542522192001343,
-      "logps/chosen": -33.281410217285156,
-      "logps/rejected": -32.49152755737305,
-      "loss": 0.2642,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.546822190284729,
-      "rewards/margins": 0.9756248593330383,
-      "rewards/rejected": -0.4288027286529541,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "grad_norm": 13.0,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.8834869861602783,
-      "logits/rejected": -1.8820085525512695,
-      "logps/chosen": -28.202373504638672,
-      "logps/rejected": -33.88237380981445,
-      "loss": 0.1975,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.5443224906921387,
-      "rewards/margins": 1.0427463054656982,
-      "rewards/rejected": -0.4984237253665924,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "grad_norm": 0.006591796875,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.9375505447387695,
-      "logits/rejected": -1.9341075420379639,
-      "logps/chosen": -30.97635269165039,
-      "logps/rejected": -35.407142639160156,
-      "loss": 0.2038,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.5803438425064087,
-      "rewards/margins": 1.0635874271392822,
-      "rewards/rejected": -0.4832436144351959,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.1414618492126465,
-      "eval_logits/rejected": -2.1367087364196777,
-      "eval_logps/chosen": -34.35294723510742,
-      "eval_logps/rejected": -38.00876235961914,
-      "eval_loss": 0.9334202408790588,
-      "eval_rewards/accuracies": 0.5689368844032288,
-      "eval_rewards/chosen": -0.12735822796821594,
-      "eval_rewards/margins": 0.06950045377016068,
-      "eval_rewards/rejected": -0.19685867428779602,
-      "eval_runtime": 145.1666,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "grad_norm": 11.0,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.7686500549316406,
-      "logits/rejected": -1.7609798908233643,
-      "logps/chosen": -32.71022033691406,
-      "logps/rejected": -35.735992431640625,
-      "loss": 0.1833,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6984082460403442,
-      "rewards/margins": 1.1687581539154053,
-      "rewards/rejected": -0.4703500270843506,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "grad_norm": 14.125,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.8466291427612305,
-      "logits/rejected": -1.8570162057876587,
-      "logps/chosen": -34.17173385620117,
-      "logps/rejected": -33.32606506347656,
-      "loss": 0.28,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6095821261405945,
-      "rewards/margins": 0.9674333333969116,
-      "rewards/rejected": -0.35785120725631714,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "grad_norm": 16.5,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.8971784114837646,
-      "logits/rejected": -1.9018369913101196,
-      "logps/chosen": -33.51736831665039,
-      "logps/rejected": -30.209964752197266,
-      "loss": 0.2911,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.5541660189628601,
-      "rewards/margins": 0.9308546781539917,
-      "rewards/rejected": -0.3766886591911316,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "grad_norm": 12.9375,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.8965809345245361,
-      "logits/rejected": -1.9047677516937256,
-      "logps/chosen": -33.255287170410156,
-      "logps/rejected": -30.980331420898438,
-      "loss": 0.2265,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.6130979061126709,
-      "rewards/margins": 1.0661498308181763,
-      "rewards/rejected": -0.4530518651008606,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "grad_norm": 13.375,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.8259570598602295,
-      "logits/rejected": -1.8164736032485962,
-      "logps/chosen": -30.39957618713379,
-      "logps/rejected": -33.05730056762695,
-      "loss": 0.2609,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.5574477314949036,
-      "rewards/margins": 0.9748461842536926,
-      "rewards/rejected": -0.4173983633518219,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "grad_norm": 11.875,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -1.9653680324554443,
-      "logits/rejected": -1.9555402994155884,
-      "logps/chosen": -24.850261688232422,
-      "logps/rejected": -31.08827781677246,
-      "loss": 0.2561,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.5495240688323975,
-      "rewards/margins": 1.035900354385376,
-      "rewards/rejected": -0.48637622594833374,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "grad_norm": 13.1875,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.8186180591583252,
-      "logits/rejected": -1.819954514503479,
-      "logps/chosen": -32.77277374267578,
-      "logps/rejected": -31.007610321044922,
-      "loss": 0.218,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6240935325622559,
-      "rewards/margins": 1.0341370105743408,
-      "rewards/rejected": -0.41004353761672974,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "grad_norm": 12.25,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.8255703449249268,
-      "logits/rejected": -1.826511025428772,
-      "logps/chosen": -31.154870986938477,
-      "logps/rejected": -34.21940994262695,
-      "loss": 0.3146,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.4937514662742615,
-      "rewards/margins": 0.9611461758613586,
-      "rewards/rejected": -0.4673948287963867,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "grad_norm": 11.6875,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.7308677434921265,
-      "logits/rejected": -1.7293297052383423,
-      "logps/chosen": -34.149600982666016,
-      "logps/rejected": -37.341552734375,
-      "loss": 0.2179,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6061262488365173,
-      "rewards/margins": 1.1918519735336304,
-      "rewards/rejected": -0.5857258439064026,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "grad_norm": 8.375,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.8834432363510132,
-      "logits/rejected": -1.8766663074493408,
-      "logps/chosen": -30.528228759765625,
-      "logps/rejected": -33.975677490234375,
-      "loss": 0.2332,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5897632837295532,
-      "rewards/margins": 1.076806664466858,
-      "rewards/rejected": -0.4870433807373047,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.139965057373047,
-      "eval_logits/rejected": -2.135190963745117,
-      "eval_logps/chosen": -34.37273025512695,
-      "eval_logps/rejected": -38.030879974365234,
-      "eval_loss": 0.9319750070571899,
-      "eval_rewards/accuracies": 0.5718438625335693,
-      "eval_rewards/chosen": -0.13527235388755798,
-      "eval_rewards/margins": 0.0704314336180687,
-      "eval_rewards/rejected": -0.20570378005504608,
-      "eval_runtime": 145.1066,
-      "eval_samples_per_second": 2.364,
-      "eval_steps_per_second": 0.296,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "grad_norm": 13.6875,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.922201156616211,
-      "logits/rejected": -1.9175071716308594,
-      "logps/chosen": -32.03234100341797,
-      "logps/rejected": -35.83629608154297,
-      "loss": 0.213,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6101375818252563,
-      "rewards/margins": 1.1332075595855713,
-      "rewards/rejected": -0.5230700373649597,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "grad_norm": 10.5625,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.8422584533691406,
-      "logits/rejected": -1.8473327159881592,
-      "logps/chosen": -33.719722747802734,
-      "logps/rejected": -33.628021240234375,
-      "loss": 0.2188,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6515445709228516,
-      "rewards/margins": 1.1027911901474,
-      "rewards/rejected": -0.4512465000152588,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "grad_norm": 9.4375,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -1.9423255920410156,
-      "logits/rejected": -1.9330682754516602,
-      "logps/chosen": -30.859844207763672,
-      "logps/rejected": -35.690673828125,
-      "loss": 0.2137,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5700357556343079,
-      "rewards/margins": 1.0691864490509033,
-      "rewards/rejected": -0.49915066361427307,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "grad_norm": 15.125,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.8686740398406982,
-      "logits/rejected": -1.8654989004135132,
-      "logps/chosen": -30.049386978149414,
-      "logps/rejected": -33.48214340209961,
-      "loss": 0.321,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.49562782049179077,
-      "rewards/margins": 0.9023343920707703,
-      "rewards/rejected": -0.40670666098594666,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "grad_norm": 4.75,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.8089139461517334,
-      "logits/rejected": -1.8163315057754517,
-      "logps/chosen": -31.528472900390625,
-      "logps/rejected": -36.56946563720703,
-      "loss": 0.3027,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.5370327234268188,
-      "rewards/margins": 0.9448447227478027,
-      "rewards/rejected": -0.4078119397163391,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "grad_norm": 10.375,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.8195326328277588,
-      "logits/rejected": -1.8125743865966797,
-      "logps/chosen": -32.26793670654297,
-      "logps/rejected": -37.91437530517578,
-      "loss": 0.2086,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.6242316961288452,
-      "rewards/margins": 1.1504604816436768,
-      "rewards/rejected": -0.5262287259101868,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "grad_norm": 12.875,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.8216333389282227,
-      "logits/rejected": -1.8221938610076904,
-      "logps/chosen": -29.730484008789062,
-      "logps/rejected": -34.02881622314453,
-      "loss": 0.2371,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.602368175983429,
-      "rewards/margins": 1.021390438079834,
-      "rewards/rejected": -0.41902226209640503,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "grad_norm": 14.0625,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.8246610164642334,
-      "logits/rejected": -1.8358711004257202,
-      "logps/chosen": -31.346548080444336,
-      "logps/rejected": -35.217620849609375,
-      "loss": 0.2028,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6920571327209473,
-      "rewards/margins": 1.1266204118728638,
-      "rewards/rejected": -0.43456321954727173,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "grad_norm": 5.0,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.8958324193954468,
-      "logits/rejected": -1.893398642539978,
-      "logps/chosen": -34.60765838623047,
-      "logps/rejected": -33.333709716796875,
-      "loss": 0.3265,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.605356752872467,
-      "rewards/margins": 0.9791496396064758,
-      "rewards/rejected": -0.3737927973270416,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "grad_norm": 12.6875,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.8685047626495361,
-      "logits/rejected": -1.8726377487182617,
-      "logps/chosen": -33.05529022216797,
-      "logps/rejected": -35.44232177734375,
-      "loss": 0.28,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5631993412971497,
-      "rewards/margins": 0.9457098245620728,
-      "rewards/rejected": -0.3825104832649231,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.139305830001831,
-      "eval_logits/rejected": -2.134549379348755,
-      "eval_logps/chosen": -34.397972106933594,
-      "eval_logps/rejected": -38.07229995727539,
-      "eval_loss": 0.9270963668823242,
-      "eval_rewards/accuracies": 0.5714285373687744,
-      "eval_rewards/chosen": -0.14536714553833008,
-      "eval_rewards/margins": 0.07690288871526718,
-      "eval_rewards/rejected": -0.22227002680301666,
-      "eval_runtime": 145.3536,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "grad_norm": 17.5,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.8788774013519287,
-      "logits/rejected": -1.8837945461273193,
-      "logps/chosen": -31.51852035522461,
-      "logps/rejected": -33.45092010498047,
-      "loss": 0.3491,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.5127608180046082,
-      "rewards/margins": 0.8846257328987122,
-      "rewards/rejected": -0.3718649446964264,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "grad_norm": 13.5,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.8375968933105469,
-      "logits/rejected": -1.8401010036468506,
-      "logps/chosen": -29.037750244140625,
-      "logps/rejected": -32.918941497802734,
-      "loss": 0.198,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.563154935836792,
-      "rewards/margins": 1.0510679483413696,
-      "rewards/rejected": -0.48791298270225525,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "grad_norm": 8.5625,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -1.8911283016204834,
-      "logits/rejected": -1.8856918811798096,
-      "logps/chosen": -32.591796875,
-      "logps/rejected": -34.6553955078125,
-      "loss": 0.2597,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5589276552200317,
-      "rewards/margins": 1.0138864517211914,
-      "rewards/rejected": -0.4549587368965149,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "grad_norm": 15.8125,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.8267465829849243,
-      "logits/rejected": -1.8307063579559326,
-      "logps/chosen": -34.41118240356445,
-      "logps/rejected": -35.5274543762207,
-      "loss": 0.252,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5370186567306519,
-      "rewards/margins": 0.9801691174507141,
-      "rewards/rejected": -0.4431503713130951,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "grad_norm": 11.9375,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.8240203857421875,
-      "logits/rejected": -1.8369216918945312,
-      "logps/chosen": -30.501928329467773,
-      "logps/rejected": -34.156166076660156,
-      "loss": 0.251,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5443831086158752,
-      "rewards/margins": 0.9988336563110352,
-      "rewards/rejected": -0.4544506072998047,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "grad_norm": 14.125,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.814233422279358,
-      "logits/rejected": -1.807539701461792,
-      "logps/chosen": -30.65509605407715,
-      "logps/rejected": -34.37706756591797,
-      "loss": 0.2287,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6016949415206909,
-      "rewards/margins": 1.0253394842147827,
-      "rewards/rejected": -0.4236446022987366,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "grad_norm": 16.375,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.9399135112762451,
-      "logits/rejected": -1.940683126449585,
-      "logps/chosen": -32.10429763793945,
-      "logps/rejected": -33.903560638427734,
-      "loss": 0.2239,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.5771129131317139,
-      "rewards/margins": 1.0006603002548218,
-      "rewards/rejected": -0.4235473573207855,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "grad_norm": 12.3125,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.8709838390350342,
-      "logits/rejected": -1.8698453903198242,
-      "logps/chosen": -32.744606018066406,
-      "logps/rejected": -34.11382293701172,
-      "loss": 0.3118,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.5380905866622925,
-      "rewards/margins": 0.906886100769043,
-      "rewards/rejected": -0.3687955439090729,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "grad_norm": 10.4375,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -1.8952579498291016,
-      "logits/rejected": -1.8877201080322266,
-      "logps/chosen": -32.717063903808594,
-      "logps/rejected": -31.973791122436523,
-      "loss": 0.239,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6035231947898865,
-      "rewards/margins": 0.9764264822006226,
-      "rewards/rejected": -0.3729034662246704,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "grad_norm": 7.875,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.8662681579589844,
-      "logits/rejected": -1.8652747869491577,
-      "logps/chosen": -33.863346099853516,
-      "logps/rejected": -37.67915344238281,
-      "loss": 0.1953,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.622026801109314,
-      "rewards/margins": 1.152605652809143,
-      "rewards/rejected": -0.5305787324905396,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.1389670372009277,
-      "eval_logits/rejected": -2.134211301803589,
-      "eval_logps/chosen": -34.42226791381836,
-      "eval_logps/rejected": -38.063568115234375,
-      "eval_loss": 0.9399018287658691,
-      "eval_rewards/accuracies": 0.5631229281425476,
-      "eval_rewards/chosen": -0.1550864279270172,
-      "eval_rewards/margins": 0.06369131803512573,
-      "eval_rewards/rejected": -0.21877776086330414,
-      "eval_runtime": 145.1823,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "grad_norm": 7.40625,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -1.9152100086212158,
-      "logits/rejected": -1.9206510782241821,
-      "logps/chosen": -34.14377975463867,
-      "logps/rejected": -36.70298767089844,
-      "loss": 0.2774,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.5293335914611816,
-      "rewards/margins": 0.9930937886238098,
-      "rewards/rejected": -0.4637601971626282,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "grad_norm": 8.5625,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.885117769241333,
-      "logits/rejected": -1.8830986022949219,
-      "logps/chosen": -31.27766990661621,
-      "logps/rejected": -34.183876037597656,
-      "loss": 0.2761,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5354140996932983,
-      "rewards/margins": 1.0158647298812866,
-      "rewards/rejected": -0.4804508090019226,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "grad_norm": 10.4375,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.7881187200546265,
-      "logits/rejected": -1.790825605392456,
-      "logps/chosen": -28.32158851623535,
-      "logps/rejected": -31.530282974243164,
-      "loss": 0.1986,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5708181262016296,
-      "rewards/margins": 1.0849167108535767,
-      "rewards/rejected": -0.514098584651947,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "grad_norm": 13.9375,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -1.886063814163208,
-      "logits/rejected": -1.882871389389038,
-      "logps/chosen": -31.68621826171875,
-      "logps/rejected": -33.42014694213867,
-      "loss": 0.249,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6070117354393005,
-      "rewards/margins": 0.9643187522888184,
-      "rewards/rejected": -0.35730695724487305,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "grad_norm": 14.4375,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.8054192066192627,
-      "logits/rejected": -1.7991243600845337,
-      "logps/chosen": -26.93368911743164,
-      "logps/rejected": -33.427791595458984,
-      "loss": 0.2445,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.4817207455635071,
-      "rewards/margins": 0.993517279624939,
-      "rewards/rejected": -0.5117965936660767,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "grad_norm": 10.75,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.9188390970230103,
-      "logits/rejected": -1.9163001775741577,
-      "logps/chosen": -31.702442169189453,
-      "logps/rejected": -36.311798095703125,
-      "loss": 0.2506,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.557451069355011,
-      "rewards/margins": 1.0455574989318848,
-      "rewards/rejected": -0.4881063997745514,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "grad_norm": 6.78125,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -1.9051227569580078,
-      "logits/rejected": -1.9030368328094482,
-      "logps/chosen": -30.17287254333496,
-      "logps/rejected": -33.75264358520508,
-      "loss": 0.1641,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.5766139030456543,
-      "rewards/margins": 1.1320630311965942,
-      "rewards/rejected": -0.5554491877555847,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "grad_norm": 12.4375,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.8981797695159912,
-      "logits/rejected": -1.8859472274780273,
-      "logps/chosen": -34.58381271362305,
-      "logps/rejected": -35.41051483154297,
-      "loss": 0.1915,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5845947861671448,
-      "rewards/margins": 1.1202127933502197,
-      "rewards/rejected": -0.535618007183075,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "grad_norm": 14.5625,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.7928569316864014,
-      "logits/rejected": -1.7875295877456665,
-      "logps/chosen": -33.692710876464844,
-      "logps/rejected": -35.34910583496094,
-      "loss": 0.1739,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6627923250198364,
-      "rewards/margins": 1.1073065996170044,
-      "rewards/rejected": -0.4445141851902008,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "grad_norm": 9.875,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.838303804397583,
-      "logits/rejected": -1.8414958715438843,
-      "logps/chosen": -31.297800064086914,
-      "logps/rejected": -32.53749084472656,
-      "loss": 0.2936,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.526917576789856,
-      "rewards/margins": 0.9474434852600098,
-      "rewards/rejected": -0.4205259382724762,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.1406960487365723,
-      "eval_logits/rejected": -2.1359305381774902,
-      "eval_logps/chosen": -34.38763427734375,
-      "eval_logps/rejected": -38.051002502441406,
-      "eval_loss": 0.9310786128044128,
-      "eval_rewards/accuracies": 0.5863787531852722,
-      "eval_rewards/chosen": -0.14123326539993286,
-      "eval_rewards/margins": 0.07251953333616257,
-      "eval_rewards/rejected": -0.21375279128551483,
-      "eval_runtime": 145.1819,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "grad_norm": 13.125,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.8620998859405518,
-      "logits/rejected": -1.857629418373108,
-      "logps/chosen": -29.83384132385254,
-      "logps/rejected": -32.17254638671875,
-      "loss": 0.1952,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.5759321451187134,
-      "rewards/margins": 1.0817142724990845,
-      "rewards/rejected": -0.5057822465896606,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "grad_norm": 10.9375,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.8760480880737305,
-      "logits/rejected": -1.870548963546753,
-      "logps/chosen": -33.693180084228516,
-      "logps/rejected": -35.85017013549805,
-      "loss": 0.201,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6323033571243286,
-      "rewards/margins": 1.1444201469421387,
-      "rewards/rejected": -0.5121166706085205,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "grad_norm": 8.75,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.8780237436294556,
-      "logits/rejected": -1.8647384643554688,
-      "logps/chosen": -31.742191314697266,
-      "logps/rejected": -35.399742126464844,
-      "loss": 0.2536,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.5407308340072632,
-      "rewards/margins": 1.0526906251907349,
-      "rewards/rejected": -0.5119598507881165,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "grad_norm": 7.96875,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.923841118812561,
-      "logits/rejected": -1.9213125705718994,
-      "logps/chosen": -33.42062759399414,
-      "logps/rejected": -32.888607025146484,
-      "loss": 0.198,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.6449113488197327,
-      "rewards/margins": 1.064406156539917,
-      "rewards/rejected": -0.41949471831321716,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "grad_norm": 7.9375,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.8493897914886475,
-      "logits/rejected": -1.846502661705017,
-      "logps/chosen": -32.840755462646484,
-      "logps/rejected": -36.272361755371094,
-      "loss": 0.1617,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.6221579909324646,
-      "rewards/margins": 1.2122505903244019,
-      "rewards/rejected": -0.5900925397872925,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "grad_norm": 9.0,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.9173166751861572,
-      "logits/rejected": -1.912136435508728,
-      "logps/chosen": -29.191509246826172,
-      "logps/rejected": -33.58539581298828,
-      "loss": 0.1642,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6277153491973877,
-      "rewards/margins": 1.1088515520095825,
-      "rewards/rejected": -0.4811362624168396,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "grad_norm": 8.75,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.943853735923767,
-      "logits/rejected": -1.9487136602401733,
-      "logps/chosen": -31.14084815979004,
-      "logps/rejected": -32.768775939941406,
-      "loss": 0.1881,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6280597448348999,
-      "rewards/margins": 1.063023567199707,
-      "rewards/rejected": -0.43496379256248474,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "grad_norm": 13.25,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.8363254070281982,
-      "logits/rejected": -1.8437111377716064,
-      "logps/chosen": -33.44670867919922,
-      "logps/rejected": -33.85987854003906,
-      "loss": 0.2191,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5842136144638062,
-      "rewards/margins": 1.0888583660125732,
-      "rewards/rejected": -0.5046447515487671,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "grad_norm": 9.3125,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.74712336063385,
-      "logits/rejected": -1.7494428157806396,
-      "logps/chosen": -31.577123641967773,
-      "logps/rejected": -37.742897033691406,
-      "loss": 0.2161,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6219089031219482,
-      "rewards/margins": 1.0973594188690186,
-      "rewards/rejected": -0.47545066475868225,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "grad_norm": 5.6875,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.7973334789276123,
-      "logits/rejected": -1.8008067607879639,
-      "logps/chosen": -31.566543579101562,
-      "logps/rejected": -33.696510314941406,
-      "loss": 0.1526,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.6600514650344849,
-      "rewards/margins": 1.1529910564422607,
-      "rewards/rejected": -0.4929395616054535,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.139777898788452,
-      "eval_logits/rejected": -2.135021686553955,
-      "eval_logps/chosen": -34.38774108886719,
-      "eval_logps/rejected": -38.054115295410156,
-      "eval_loss": 0.9307055473327637,
-      "eval_rewards/accuracies": 0.5863787531852722,
-      "eval_rewards/chosen": -0.14127586781978607,
-      "eval_rewards/margins": 0.07372146099805832,
-      "eval_rewards/rejected": -0.2149973064661026,
-      "eval_runtime": 145.1112,
-      "eval_samples_per_second": 2.364,
-      "eval_steps_per_second": 0.296,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "grad_norm": 8.0625,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.9071658849716187,
-      "logits/rejected": -1.886819839477539,
-      "logps/chosen": -29.60116195678711,
-      "logps/rejected": -36.554786682128906,
-      "loss": 0.2053,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6048943400382996,
-      "rewards/margins": 1.1602458953857422,
-      "rewards/rejected": -0.5553516149520874,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "grad_norm": 13.375,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -1.8822952508926392,
-      "logits/rejected": -1.888043761253357,
-      "logps/chosen": -31.35972023010254,
-      "logps/rejected": -33.60492706298828,
-      "loss": 0.1474,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6696279048919678,
-      "rewards/margins": 1.115299105644226,
-      "rewards/rejected": -0.4456712603569031,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "grad_norm": 7.28125,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.8583717346191406,
-      "logits/rejected": -1.8534027338027954,
-      "logps/chosen": -31.66836166381836,
-      "logps/rejected": -35.97147750854492,
-      "loss": 0.1801,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5416578650474548,
-      "rewards/margins": 1.1013325452804565,
-      "rewards/rejected": -0.5596746802330017,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "grad_norm": 10.5625,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.720171570777893,
-      "logits/rejected": -1.729686975479126,
-      "logps/chosen": -32.17263412475586,
-      "logps/rejected": -31.95621681213379,
-      "loss": 0.2302,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6507819294929504,
-      "rewards/margins": 1.019622564315796,
-      "rewards/rejected": -0.368840754032135,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "grad_norm": 9.25,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.7789561748504639,
-      "logits/rejected": -1.7837011814117432,
-      "logps/chosen": -32.18306350708008,
-      "logps/rejected": -34.33165740966797,
-      "loss": 0.1217,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.7071508169174194,
-      "rewards/margins": 1.157888650894165,
-      "rewards/rejected": -0.45073771476745605,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "grad_norm": 17.25,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.8064203262329102,
-      "logits/rejected": -1.811870813369751,
-      "logps/chosen": -32.34109878540039,
-      "logps/rejected": -35.7831916809082,
-      "loss": 0.2097,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.5962255597114563,
-      "rewards/margins": 1.0842751264572144,
-      "rewards/rejected": -0.4880496561527252,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "grad_norm": 7.28125,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -1.8998634815216064,
-      "logits/rejected": -1.8953298330307007,
-      "logps/chosen": -31.270008087158203,
-      "logps/rejected": -32.123897552490234,
-      "loss": 0.1836,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.546270489692688,
-      "rewards/margins": 1.0310702323913574,
-      "rewards/rejected": -0.4847997725009918,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "grad_norm": 14.0625,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.8143882751464844,
-      "logits/rejected": -1.816428542137146,
-      "logps/chosen": -28.297687530517578,
-      "logps/rejected": -31.918853759765625,
-      "loss": 0.2172,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.5446463823318481,
-      "rewards/margins": 1.0145161151885986,
-      "rewards/rejected": -0.4698697626590729,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "grad_norm": 13.75,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.8110907077789307,
-      "logits/rejected": -1.8157243728637695,
-      "logps/chosen": -30.599262237548828,
-      "logps/rejected": -33.90684127807617,
-      "loss": 0.228,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5680925250053406,
-      "rewards/margins": 0.9356026649475098,
-      "rewards/rejected": -0.3675101101398468,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "grad_norm": 5.28125,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.8766982555389404,
-      "logits/rejected": -1.8799679279327393,
-      "logps/chosen": -33.26051712036133,
-      "logps/rejected": -34.686241149902344,
-      "loss": 0.1121,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.6680229902267456,
-      "rewards/margins": 1.1730940341949463,
-      "rewards/rejected": -0.5050708651542664,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.14040207862854,
-      "eval_logits/rejected": -2.135646104812622,
-      "eval_logps/chosen": -34.378780364990234,
-      "eval_logps/rejected": -38.054847717285156,
-      "eval_loss": 0.925031840801239,
-      "eval_rewards/accuracies": 0.6038206219673157,
-      "eval_rewards/chosen": -0.13769178092479706,
-      "eval_rewards/margins": 0.07760081440210342,
-      "eval_rewards/rejected": -0.21529260277748108,
-      "eval_runtime": 145.1344,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "grad_norm": 13.4375,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -1.945010781288147,
-      "logits/rejected": -1.9381306171417236,
-      "logps/chosen": -32.14293670654297,
-      "logps/rejected": -35.293968200683594,
-      "loss": 0.1774,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.6638438105583191,
-      "rewards/margins": 1.0836031436920166,
-      "rewards/rejected": -0.4197593331336975,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "grad_norm": 12.1875,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.8345167636871338,
-      "logits/rejected": -1.8454965353012085,
-      "logps/chosen": -32.55244827270508,
-      "logps/rejected": -34.81972122192383,
-      "loss": 0.1625,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6387423276901245,
-      "rewards/margins": 1.1301978826522827,
-      "rewards/rejected": -0.49145546555519104,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "grad_norm": 13.0,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.9138675928115845,
-      "logits/rejected": -1.9257293939590454,
-      "logps/chosen": -33.43199920654297,
-      "logps/rejected": -34.919532775878906,
-      "loss": 0.1706,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6515201926231384,
-      "rewards/margins": 1.1639997959136963,
-      "rewards/rejected": -0.5124797224998474,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "grad_norm": 11.1875,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.904793381690979,
-      "logits/rejected": -1.9069957733154297,
-      "logps/chosen": -31.7263126373291,
-      "logps/rejected": -35.0744743347168,
-      "loss": 0.2078,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.5928308367729187,
-      "rewards/margins": 1.0786316394805908,
-      "rewards/rejected": -0.4858008325099945,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "grad_norm": 9.5625,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.8884856700897217,
-      "logits/rejected": -1.8813756704330444,
-      "logps/chosen": -33.934364318847656,
-      "logps/rejected": -34.85586929321289,
-      "loss": 0.2029,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.6535843014717102,
-      "rewards/margins": 1.0877697467803955,
-      "rewards/rejected": -0.43418556451797485,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "grad_norm": 18.875,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.8547000885009766,
-      "logits/rejected": -1.860727310180664,
-      "logps/chosen": -29.743526458740234,
-      "logps/rejected": -34.97495651245117,
-      "loss": 0.1719,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.5878596305847168,
-      "rewards/margins": 1.1098417043685913,
-      "rewards/rejected": -0.5219820737838745,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "grad_norm": 12.875,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.8023372888565063,
-      "logits/rejected": -1.8003690242767334,
-      "logps/chosen": -29.268962860107422,
-      "logps/rejected": -31.533855438232422,
-      "loss": 0.2267,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.6306362152099609,
-      "rewards/margins": 1.006486177444458,
-      "rewards/rejected": -0.37585002183914185,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "grad_norm": 9.5625,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -1.9781367778778076,
-      "logits/rejected": -1.9725608825683594,
-      "logps/chosen": -33.99125671386719,
-      "logps/rejected": -33.27691650390625,
-      "loss": 0.1815,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.5176017880439758,
-      "rewards/margins": 1.0056849718093872,
-      "rewards/rejected": -0.4880831241607666,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "grad_norm": 14.25,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.8709309101104736,
-      "logits/rejected": -1.873234510421753,
-      "logps/chosen": -27.13213539123535,
-      "logps/rejected": -29.021121978759766,
-      "loss": 0.2587,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.5376989245414734,
-      "rewards/margins": 0.9372351765632629,
-      "rewards/rejected": -0.39953625202178955,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "grad_norm": 16.25,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.8781601190567017,
-      "logits/rejected": -1.8725292682647705,
-      "logps/chosen": -31.48541831970215,
-      "logps/rejected": -33.22844696044922,
-      "loss": 0.215,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5261479020118713,
-      "rewards/margins": 1.0299475193023682,
-      "rewards/rejected": -0.5037996768951416,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.140532970428467,
-      "eval_logits/rejected": -2.135777473449707,
-      "eval_logps/chosen": -34.38066101074219,
-      "eval_logps/rejected": -38.052310943603516,
-      "eval_loss": 0.9279635548591614,
-      "eval_rewards/accuracies": 0.5805647969245911,
-      "eval_rewards/chosen": -0.1384437084197998,
-      "eval_rewards/margins": 0.07583153247833252,
-      "eval_rewards/rejected": -0.21427522599697113,
-      "eval_runtime": 145.1697,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "grad_norm": 16.375,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.803832769393921,
-      "logits/rejected": -1.8080018758773804,
-      "logps/chosen": -32.76865005493164,
-      "logps/rejected": -35.45244216918945,
-      "loss": 0.2319,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.56093829870224,
-      "rewards/margins": 0.9777617454528809,
-      "rewards/rejected": -0.41682347655296326,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "grad_norm": 5.53125,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.8591749668121338,
-      "logits/rejected": -1.8575620651245117,
-      "logps/chosen": -29.235681533813477,
-      "logps/rejected": -33.72986602783203,
-      "loss": 0.2297,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.5993990302085876,
-      "rewards/margins": 1.0554559230804443,
-      "rewards/rejected": -0.4560568928718567,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "grad_norm": 14.1875,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -1.9594846963882446,
-      "logits/rejected": -1.951703667640686,
-      "logps/chosen": -26.725296020507812,
-      "logps/rejected": -29.879215240478516,
-      "loss": 0.2118,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.5350590944290161,
-      "rewards/margins": 1.0181024074554443,
-      "rewards/rejected": -0.4830433428287506,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 13.9375,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.8623542785644531,
-      "logits/rejected": -1.8520355224609375,
-      "logps/chosen": -31.616281509399414,
-      "logps/rejected": -36.66792297363281,
-      "loss": 0.1728,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.5799667835235596,
-      "rewards/margins": 1.0826661586761475,
-      "rewards/rejected": -0.5026994943618774,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.2625940967689861,
-      "train_runtime": 10748.6274,
-      "train_samples_per_second": 1.146,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.6506414413452148,
+      "train_runtime": 3254.0644,
+      "train_samples_per_second": 0.946,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,