diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.3086,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2537 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8665881156921387,
-      "logits/rejected": -1.8709055185317993,
-      "logps/chosen": -36.99662399291992,
-      "logps/rejected": -33.65571594238281,
-      "loss": 0.3052,
-      "rewards/accuracies": 0.5416666865348816,
-      "rewards/chosen": 0.008980684913694859,
-      "rewards/margins": 0.032059140503406525,
-      "rewards/rejected": -0.02307845838367939,
+      "logits/chosen": -1.8667426109313965,
+      "logits/rejected": -1.8710602521896362,
+      "logps/chosen": -36.991912841796875,
+      "logps/rejected": -33.67206954956055,
+      "loss": 0.9773,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.0058750128373503685,
+      "rewards/margins": 0.022671451792120934,
+      "rewards/rejected": -0.01679643802344799,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9977455139160156,
-      "logits/rejected": -2.000382423400879,
-      "logps/chosen": -29.642925262451172,
-      "logps/rejected": -29.056737899780273,
-      "loss": 0.3525,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0006573178106918931,
-      "rewards/margins": -0.00669272243976593,
-      "rewards/rejected": 0.006035405211150646,
+      "logits/chosen": -1.9977442026138306,
+      "logits/rejected": -2.0003952980041504,
+      "logps/chosen": -29.659366607666016,
+      "logps/rejected": -29.05437660217285,
+      "loss": 1.0105,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.006868218071758747,
+      "rewards/margins": -0.010493903420865536,
+      "rewards/rejected": 0.003625686513260007,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9204607009887695,
-      "logits/rejected": -1.9177772998809814,
-      "logps/chosen": -31.42336654663086,
-      "logps/rejected": -33.22785568237305,
-      "loss": 0.3603,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.006546213291585445,
-      "rewards/margins": 0.0019128695130348206,
-      "rewards/rejected": -0.00845908559858799,
+      "logits/chosen": -1.920693039894104,
+      "logits/rejected": -1.91802179813385,
+      "logps/chosen": -31.39971351623535,
+      "logps/rejected": -33.21495819091797,
+      "loss": 0.9948,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.006551843136548996,
+      "rewards/margins": 0.0051523735746741295,
+      "rewards/rejected": 0.001399471191689372,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0172362327575684,
-      "logits/rejected": -2.008507251739502,
-      "logps/chosen": -32.56964874267578,
-      "logps/rejected": -32.50572967529297,
-      "loss": 0.3558,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.006439635064452887,
-      "rewards/margins": 0.00019515231542754918,
-      "rewards/rejected": 0.006244482938200235,
+      "logits/chosen": -2.018057107925415,
+      "logits/rejected": -2.0093047618865967,
+      "logps/chosen": -32.565284729003906,
+      "logps/rejected": -32.50053405761719,
+      "loss": 1.0002,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0046076299622654915,
+      "rewards/margins": -0.00024683662923052907,
+      "rewards/rejected": 0.004854466766119003,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8618619441986084,
-      "logits/rejected": -1.8510783910751343,
-      "logps/chosen": -33.56026077270508,
-      "logps/rejected": -35.45254898071289,
-      "loss": 0.3691,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.0025013976264744997,
-      "rewards/margins": -0.002315213903784752,
-      "rewards/rejected": -0.00018618404283188283,
+      "logits/chosen": -1.8627817630767822,
+      "logits/rejected": -1.851999044418335,
+      "logps/chosen": -33.549964904785156,
+      "logps/rejected": -35.44340896606445,
+      "loss": 1.0006,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0030060340650379658,
+      "rewards/margins": -0.0005688609671778977,
+      "rewards/rejected": 0.003574896603822708,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9393202066421509,
-      "logits/rejected": -1.9412600994110107,
-      "logps/chosen": -32.57838439941406,
-      "logps/rejected": -33.215576171875,
-      "loss": 0.3063,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.01962057128548622,
-      "rewards/margins": 0.05940054729580879,
-      "rewards/rejected": -0.03977997973561287,
+      "logits/chosen": -1.9416770935058594,
+      "logits/rejected": -1.9436094760894775,
+      "logps/chosen": -32.53351593017578,
+      "logps/rejected": -33.217529296875,
+      "loss": 0.9561,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.026666466146707535,
+      "rewards/margins": 0.04512657970190048,
+      "rewards/rejected": -0.018460111692547798,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0718436241149902,
-      "logits/rejected": -2.0768017768859863,
-      "logps/chosen": -33.97806167602539,
-      "logps/rejected": -36.63082504272461,
-      "loss": 0.4257,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00938049890100956,
-      "rewards/margins": 0.06513925641775131,
-      "rewards/rejected": -0.0557587556540966,
+      "logits/chosen": -2.072779655456543,
+      "logits/rejected": -2.077756881713867,
+      "logps/chosen": -34.002342224121094,
+      "logps/rejected": -36.633216857910156,
+      "loss": 0.9798,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0055419160053133965,
+      "rewards/margins": 0.020196745172142982,
+      "rewards/rejected": -0.025738662108778954,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9351739883422852,
-      "logits/rejected": -1.9383188486099243,
-      "logps/chosen": -34.33073043823242,
-      "logps/rejected": -34.61904525756836,
-      "loss": 0.2902,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.08736880123615265,
-      "rewards/margins": 0.11134655773639679,
-      "rewards/rejected": -0.023977745324373245,
+      "logits/chosen": -1.9332294464111328,
+      "logits/rejected": -1.9363486766815186,
+      "logps/chosen": -34.33064651489258,
+      "logps/rejected": -34.64745330810547,
+      "loss": 0.9391,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03886651247739792,
+      "rewards/margins": 0.06088464334607124,
+      "rewards/rejected": -0.022018127143383026,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.944392204284668,
-      "logits/rejected": -1.9489190578460693,
-      "logps/chosen": -32.419586181640625,
-      "logps/rejected": -32.3698844909668,
-      "loss": 0.3611,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.043051257729530334,
-      "rewards/margins": 0.052930813282728195,
-      "rewards/rejected": -0.009879561141133308,
+      "logits/chosen": -1.9409536123275757,
+      "logits/rejected": -1.9454677104949951,
+      "logps/chosen": -32.381752014160156,
+      "logps/rejected": -32.35805892944336,
+      "loss": 0.9661,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0342683270573616,
+      "rewards/margins": 0.033929694443941116,
+      "rewards/rejected": 0.0003386303724255413,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0419769287109375,
-      "logits/rejected": -2.0399627685546875,
-      "logps/chosen": -32.174407958984375,
-      "logps/rejected": -31.26608657836914,
-      "loss": 0.3038,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.059785228222608566,
-      "rewards/margins": 0.05967242643237114,
-      "rewards/rejected": 0.00011279433965682983,
+      "logits/chosen": -2.0382819175720215,
+      "logits/rejected": -2.036304473876953,
+      "logps/chosen": -32.15763473510742,
+      "logps/rejected": -31.302764892578125,
+      "loss": 0.9521,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.033279187977313995,
+      "rewards/margins": 0.047900475561618805,
+      "rewards/rejected": -0.014621290378272533,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2372143268585205,
-      "eval_logits/rejected": -2.2323503494262695,
-      "eval_logps/chosen": -34.01276779174805,
-      "eval_logps/rejected": -37.51152420043945,
-      "eval_loss": 0.3511974811553955,
-      "eval_rewards/accuracies": 0.5423588156700134,
-      "eval_rewards/chosen": 0.01960929110646248,
-      "eval_rewards/margins": 0.01502405758947134,
-      "eval_rewards/rejected": 0.004585230257362127,
-      "eval_runtime": 145.9032,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2327499389648438,
+      "eval_logits/rejected": -2.227902889251709,
+      "eval_logps/chosen": -34.038063049316406,
+      "eval_logps/rejected": -37.54012680053711,
+      "eval_loss": 0.9920137524604797,
+      "eval_rewards/accuracies": 0.5361295938491821,
+      "eval_rewards/chosen": -0.0014053798513486981,
+      "eval_rewards/margins": 0.007998107932507992,
+      "eval_rewards/rejected": -0.009403487667441368,
+      "eval_runtime": 146.1087,
+      "eval_samples_per_second": 2.348,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.998891830444336,
-      "logits/rejected": -1.996492624282837,
-      "logps/chosen": -33.14598083496094,
-      "logps/rejected": -34.020729064941406,
-      "loss": 0.4689,
+      "logits/chosen": -1.9931761026382446,
+      "logits/rejected": -1.9907801151275635,
+      "logps/chosen": -33.120845794677734,
+      "logps/rejected": -34.02234649658203,
+      "loss": 0.9536,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.08805312216281891,
-      "rewards/margins": 0.08027410507202148,
-      "rewards/rejected": 0.007779018487781286,
+      "rewards/chosen": 0.04918808490037918,
+      "rewards/margins": 0.046377379447221756,
+      "rewards/rejected": 0.0028106991667300463,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.010932445526123,
-      "logits/rejected": -2.0025696754455566,
-      "logps/chosen": -32.37172317504883,
-      "logps/rejected": -32.118797302246094,
-      "loss": 0.4465,
+      "logits/chosen": -2.004035472869873,
+      "logits/rejected": -1.9957201480865479,
+      "logps/chosen": -32.31081771850586,
+      "logps/rejected": -32.129127502441406,
+      "loss": 0.9593,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.06586603820323944,
-      "rewards/margins": 0.02742874063551426,
-      "rewards/rejected": 0.03843729570508003,
+      "rewards/chosen": 0.05363499000668526,
+      "rewards/margins": 0.04068244248628616,
+      "rewards/rejected": 0.012952548451721668,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0387539863586426,
-      "logits/rejected": -2.030724287033081,
-      "logps/chosen": -30.41655921936035,
-      "logps/rejected": -32.060333251953125,
-      "loss": 0.3844,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.05324209854006767,
-      "rewards/margins": 0.05983499437570572,
-      "rewards/rejected": -0.00659290561452508,
+      "logits/chosen": -2.0312001705169678,
+      "logits/rejected": -2.023239850997925,
+      "logps/chosen": -30.317270278930664,
+      "logps/rejected": -32.05809783935547,
+      "loss": 0.9384,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06337819993495941,
+      "rewards/margins": 0.06541591882705688,
+      "rewards/rejected": -0.0020377314649522305,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.967858076095581,
-      "logits/rejected": -1.9781148433685303,
-      "logps/chosen": -31.223413467407227,
-      "logps/rejected": -32.55517578125,
-      "loss": 0.3904,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.1766217201948166,
-      "rewards/margins": 0.1948881596326828,
-      "rewards/rejected": -0.01826643943786621,
+      "logits/chosen": -1.9611848592758179,
+      "logits/rejected": -1.9714059829711914,
+      "logps/chosen": -31.235523223876953,
+      "logps/rejected": -32.55936813354492,
+      "loss": 0.9165,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.07365532219409943,
+      "rewards/margins": 0.08345074951648712,
+      "rewards/rejected": -0.009795431979000568,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.880910873413086,
-      "logits/rejected": -1.8820507526397705,
-      "logps/chosen": -34.01464080810547,
-      "logps/rejected": -34.783546447753906,
-      "loss": 0.3871,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.160243421792984,
-      "rewards/margins": 0.1839032918214798,
-      "rewards/rejected": -0.023659853264689445,
+      "logits/chosen": -1.8718490600585938,
+      "logits/rejected": -1.8730093240737915,
+      "logps/chosen": -33.873046875,
+      "logps/rejected": -34.80980682373047,
+      "loss": 0.8563,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.12785716354846954,
+      "rewards/margins": 0.1488770991563797,
+      "rewards/rejected": -0.021019931882619858,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.933895468711853,
-      "logits/rejected": -1.9304730892181396,
-      "logps/chosen": -36.02853775024414,
-      "logps/rejected": -32.699058532714844,
-      "loss": 0.2729,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.1288261115550995,
-      "rewards/margins": 0.10750452429056168,
-      "rewards/rejected": 0.021321602165699005,
+      "logits/chosen": -1.9235217571258545,
+      "logits/rejected": -1.9201278686523438,
+      "logps/chosen": -36.01029968261719,
+      "logps/rejected": -32.690834045410156,
+      "loss": 0.9482,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.06455211341381073,
+      "rewards/margins": 0.05178683251142502,
+      "rewards/rejected": 0.012765283696353436,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0341715812683105,
-      "logits/rejected": -2.0267820358276367,
-      "logps/chosen": -33.55347442626953,
-      "logps/rejected": -31.3526554107666,
-      "loss": 0.294,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.21994857490062714,
-      "rewards/margins": 0.22067300975322723,
-      "rewards/rejected": -0.0007244400912895799,
+      "logits/chosen": -2.0244345664978027,
+      "logits/rejected": -2.0171027183532715,
+      "logps/chosen": -33.474937438964844,
+      "logps/rejected": -31.42409324645996,
+      "loss": 0.8419,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.12916772067546844,
+      "rewards/margins": 0.15806543827056885,
+      "rewards/rejected": -0.028897713869810104,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0400891304016113,
-      "logits/rejected": -2.045360565185547,
-      "logps/chosen": -32.370338439941406,
-      "logps/rejected": -32.4719123840332,
-      "loss": 0.2829,
+      "logits/chosen": -2.0310873985290527,
+      "logits/rejected": -2.036334276199341,
+      "logps/chosen": -32.2062873840332,
+      "logps/rejected": -32.42302703857422,
+      "loss": 0.8919,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.15700635313987732,
-      "rewards/margins": 0.145101398229599,
-      "rewards/rejected": 0.011904975399374962,
+      "rewards/chosen": 0.1354007124900818,
+      "rewards/margins": 0.11055666208267212,
+      "rewards/rejected": 0.024844054132699966,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.041393280029297,
-      "logits/rejected": -2.038623809814453,
-      "logps/chosen": -31.328174591064453,
-      "logps/rejected": -31.316492080688477,
-      "loss": 0.3044,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.14470075070858002,
-      "rewards/margins": 0.1431477963924408,
-      "rewards/rejected": 0.0015529401134699583,
+      "logits/chosen": -2.0314042568206787,
+      "logits/rejected": -2.02862286567688,
+      "logps/chosen": -31.278457641601562,
+      "logps/rejected": -31.348251342773438,
+      "loss": 0.9038,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0841989517211914,
+      "rewards/margins": 0.09621445834636688,
+      "rewards/rejected": -0.012015508487820625,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9122215509414673,
-      "logits/rejected": -1.9168663024902344,
-      "logps/chosen": -31.424020767211914,
-      "logps/rejected": -32.784080505371094,
-      "loss": 0.3157,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.16403506696224213,
-      "rewards/margins": 0.1578713059425354,
-      "rewards/rejected": 0.0061637843027710915,
+      "logits/chosen": -1.9020229578018188,
+      "logits/rejected": -1.906660795211792,
+      "logps/chosen": -31.28672218322754,
+      "logps/rejected": -32.84270477294922,
+      "loss": 0.8515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12782469391822815,
+      "rewards/margins": 0.14853176474571228,
+      "rewards/rejected": -0.020707078278064728,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.237051010131836,
-      "eval_logits/rejected": -2.2321863174438477,
-      "eval_logps/chosen": -34.018070220947266,
-      "eval_logps/rejected": -37.51838684082031,
-      "eval_loss": 0.3716273605823517,
-      "eval_rewards/accuracies": 0.5245016813278198,
-      "eval_rewards/chosen": 0.014834923669695854,
-      "eval_rewards/margins": 0.016425320878624916,
-      "eval_rewards/rejected": -0.0015903981402516365,
-      "eval_runtime": 145.5151,
-      "eval_samples_per_second": 2.357,
+      "eval_logits/chosen": -2.22743558883667,
+      "eval_logits/rejected": -2.2226204872131348,
+      "eval_logps/chosen": -34.04505920410156,
+      "eval_logps/rejected": -37.573036193847656,
+      "eval_loss": 0.9816663265228271,
+      "eval_rewards/accuracies": 0.559385359287262,
+      "eval_rewards/chosen": -0.004204160068184137,
+      "eval_rewards/margins": 0.018361244350671768,
+      "eval_rewards/rejected": -0.022565403953194618,
+      "eval_runtime": 145.491,
+      "eval_samples_per_second": 2.358,
       "eval_steps_per_second": 0.296,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.023789882659912,
-      "logits/rejected": -2.034484386444092,
-      "logps/chosen": -31.767370223999023,
-      "logps/rejected": -33.890621185302734,
-      "loss": 0.275,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18354059755802155,
-      "rewards/margins": 0.20867136120796204,
-      "rewards/rejected": -0.025130782276391983,
+      "logits/chosen": -2.0146515369415283,
+      "logits/rejected": -2.025285005569458,
+      "logps/chosen": -31.747217178344727,
+      "logps/rejected": -33.96654510498047,
+      "loss": 0.8755,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.08963600546121597,
+      "rewards/margins": 0.13117524981498718,
+      "rewards/rejected": -0.04153924435377121,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.916685700416565,
-      "logits/rejected": -1.9314892292022705,
-      "logps/chosen": -29.956628799438477,
-      "logps/rejected": -31.564035415649414,
-      "loss": 0.2958,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13942097127437592,
-      "rewards/margins": 0.14469322562217712,
-      "rewards/rejected": -0.005272268317639828,
+      "logits/chosen": -1.9072997570037842,
+      "logits/rejected": -1.922041654586792,
+      "logps/chosen": -29.81575584411621,
+      "logps/rejected": -31.636306762695312,
+      "loss": 0.8509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11831430345773697,
+      "rewards/margins": 0.14956556260585785,
+      "rewards/rejected": -0.03125125169754028,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9737945795059204,
-      "logits/rejected": -1.9777710437774658,
-      "logps/chosen": -33.19129180908203,
-      "logps/rejected": -31.5566463470459,
-      "loss": 0.287,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.20355579257011414,
-      "rewards/margins": 0.21016716957092285,
-      "rewards/rejected": -0.0066113718785345554,
+      "logits/chosen": -1.9637253284454346,
+      "logits/rejected": -1.9677000045776367,
+      "logps/chosen": -33.084877014160156,
+      "logps/rejected": -31.639108657836914,
+      "loss": 0.8476,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.13303686678409576,
+      "rewards/margins": 0.16895917057991028,
+      "rewards/rejected": -0.03592229634523392,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9743419885635376,
-      "logits/rejected": -1.9523779153823853,
-      "logps/chosen": -33.9401969909668,
-      "logps/rejected": -35.008758544921875,
-      "loss": 0.3185,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1921955645084381,
-      "rewards/margins": 0.24436470866203308,
-      "rewards/rejected": -0.052169155329465866,
+      "logits/chosen": -1.9612106084823608,
+      "logits/rejected": -1.9394094944000244,
+      "logps/chosen": -33.84415817260742,
+      "logps/rejected": -35.132205963134766,
+      "loss": 0.8154,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1238357201218605,
+      "rewards/margins": 0.19639968872070312,
+      "rewards/rejected": -0.07256398350000381,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0161709785461426,
-      "logits/rejected": -2.0128414630889893,
-      "logps/chosen": -32.74829864501953,
-      "logps/rejected": -36.236392974853516,
-      "loss": 0.2847,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.15909257531166077,
-      "rewards/margins": 0.1694144755601883,
-      "rewards/rejected": -0.010321905836462975,
+      "logits/chosen": -2.002776861190796,
+      "logits/rejected": -1.999464988708496,
+      "logps/chosen": -32.74934387207031,
+      "logps/rejected": -36.28064727783203,
+      "loss": 0.9096,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0702909380197525,
+      "rewards/margins": 0.09257940202951431,
+      "rewards/rejected": -0.02228846587240696,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8847742080688477,
-      "logits/rejected": -1.8823268413543701,
-      "logps/chosen": -34.01182174682617,
-      "logps/rejected": -35.481346130371094,
-      "loss": 0.3073,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1589193046092987,
-      "rewards/margins": 0.13797220587730408,
-      "rewards/rejected": 0.020947108045220375,
+      "logits/chosen": -1.8692924976348877,
+      "logits/rejected": -1.8668625354766846,
+      "logps/chosen": -33.982818603515625,
+      "logps/rejected": -35.535736083984375,
+      "loss": 0.9053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.08223171532154083,
+      "rewards/margins": 0.09467832744121552,
+      "rewards/rejected": -0.012446624226868153,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8693536520004272,
-      "logits/rejected": -1.866838812828064,
-      "logps/chosen": -34.20549392700195,
-      "logps/rejected": -31.726673126220703,
-      "loss": 0.2733,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17289286851882935,
-      "rewards/margins": 0.1327010840177536,
-      "rewards/rejected": 0.040191780775785446,
+      "logits/chosen": -1.8543907403945923,
+      "logits/rejected": -1.8520078659057617,
+      "logps/chosen": -34.20638656616211,
+      "logps/rejected": -31.873388290405273,
+      "loss": 0.8923,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07648433744907379,
+      "rewards/margins": 0.11730633676052094,
+      "rewards/rejected": -0.040821999311447144,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9730831384658813,
-      "logits/rejected": -1.962480902671814,
-      "logps/chosen": -35.06049728393555,
-      "logps/rejected": -31.792781829833984,
-      "loss": 0.2523,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2665289342403412,
-      "rewards/margins": 0.22314274311065674,
-      "rewards/rejected": 0.04338619112968445,
+      "logits/chosen": -1.9576927423477173,
+      "logits/rejected": -1.9472328424453735,
+      "logps/chosen": -34.999000549316406,
+      "logps/rejected": -31.88382911682129,
+      "loss": 0.8398,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.1430555135011673,
+      "rewards/margins": 0.16019006073474884,
+      "rewards/rejected": -0.017134560272097588,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0680813789367676,
-      "logits/rejected": -2.053079128265381,
-      "logps/chosen": -30.738479614257812,
-      "logps/rejected": -32.61243438720703,
-      "loss": 0.3194,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.16707859933376312,
-      "rewards/margins": 0.13161785900592804,
-      "rewards/rejected": 0.035460732877254486,
+      "logits/chosen": -2.0523831844329834,
+      "logits/rejected": -2.037504196166992,
+      "logps/chosen": -30.712234497070312,
+      "logps/rejected": -32.659637451171875,
+      "loss": 0.9146,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.08475608378648758,
+      "rewards/margins": 0.08787757158279419,
+      "rewards/rejected": -0.0031214915215969086,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9398882389068604,
-      "logits/rejected": -1.9373395442962646,
-      "logps/chosen": -32.6181640625,
-      "logps/rejected": -30.843700408935547,
-      "loss": 0.2156,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2780519425868988,
-      "rewards/margins": 0.3022567331790924,
-      "rewards/rejected": -0.024204757064580917,
+      "logits/chosen": -1.9236303567886353,
+      "logits/rejected": -1.9211324453353882,
+      "logps/chosen": -32.370330810546875,
+      "logps/rejected": -30.952068328857422,
+      "loss": 0.7473,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.2227112501859665,
+      "rewards/margins": 0.27681466937065125,
+      "rewards/rejected": -0.054103411734104156,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2364187240600586,
-      "eval_logits/rejected": -2.2315518856048584,
-      "eval_logps/chosen": -34.01428985595703,
-      "eval_logps/rejected": -37.496952056884766,
-      "eval_loss": 0.38450533151626587,
-      "eval_rewards/accuracies": 0.49335551261901855,
-      "eval_rewards/chosen": 0.018236981704831123,
-      "eval_rewards/margins": 0.0005384809919632971,
-      "eval_rewards/rejected": 0.01769850216805935,
-      "eval_runtime": 145.7485,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.223449468612671,
+      "eval_logits/rejected": -2.2186241149902344,
+      "eval_logps/chosen": -34.09219741821289,
+      "eval_logps/rejected": -37.61648178100586,
+      "eval_loss": 0.9830461740493774,
+      "eval_rewards/accuracies": 0.5390365719795227,
+      "eval_rewards/chosen": -0.023058738559484482,
+      "eval_rewards/margins": 0.016884688287973404,
+      "eval_rewards/rejected": -0.03994342312216759,
+      "eval_runtime": 145.6848,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 107.0,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -2.0731887817382812,
-      "logits/rejected": -2.060344934463501,
-      "logps/chosen": -32.24032211303711,
-      "logps/rejected": -32.77452087402344,
-      "loss": 0.2321,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2592628598213196,
-      "rewards/margins": 0.3793802857398987,
-      "rewards/rejected": -0.12011740356683731,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.908721923828125,
+      "logits/rejected": -1.9054863452911377,
+      "logps/chosen": -31.30953025817871,
+      "logps/rejected": -33.8243408203125,
+      "loss": 0.8639,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1175277829170227,
+      "rewards/margins": 0.15040358901023865,
+      "rewards/rejected": -0.032875802367925644,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 37.0,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9875545501708984,
-      "logits/rejected": -1.978822112083435,
-      "logps/chosen": -32.03451919555664,
-      "logps/rejected": -30.3249454498291,
-      "loss": 0.2062,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.288422554731369,
-      "rewards/margins": 0.3636184334754944,
-      "rewards/rejected": -0.07519589364528656,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.958335518836975,
+      "logits/rejected": -1.946170449256897,
+      "logps/chosen": -34.308921813964844,
+      "logps/rejected": -33.70027542114258,
+      "loss": 0.8275,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.11486158519983292,
+      "rewards/margins": 0.17943526804447174,
+      "rewards/rejected": -0.06457368284463882,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 25.75,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.9262025356292725,
-      "logits/rejected": -1.9383175373077393,
-      "logps/chosen": -30.169696807861328,
-      "logps/rejected": -33.530517578125,
-      "loss": 0.214,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2786136865615845,
-      "rewards/margins": 0.3940487504005432,
-      "rewards/rejected": -0.11543502658605576,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -1.9931294918060303,
+      "logits/rejected": -1.9917207956314087,
+      "logps/chosen": -33.1971321105957,
+      "logps/rejected": -32.582763671875,
+      "loss": 0.8474,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.11546528339385986,
+      "rewards/margins": 0.15673772990703583,
+      "rewards/rejected": -0.04127243906259537,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 52.75,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -1.8931338787078857,
-      "logits/rejected": -1.8838812112808228,
-      "logps/chosen": -34.24372100830078,
-      "logps/rejected": -35.92654037475586,
-      "loss": 0.3165,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.40968865156173706,
-      "rewards/margins": 0.47252458333969116,
-      "rewards/rejected": -0.06283595412969589,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.0800864696502686,
+      "logits/rejected": -2.064441204071045,
+      "logps/chosen": -33.79554748535156,
+      "logps/rejected": -33.09272384643555,
+      "loss": 0.8566,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.15353715419769287,
+      "rewards/margins": 0.14341332018375397,
+      "rewards/rejected": 0.010123846121132374,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 38.75,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.9516513347625732,
-      "logits/rejected": -1.9522939920425415,
-      "logps/chosen": -34.0687141418457,
-      "logps/rejected": -33.91803741455078,
-      "loss": 0.2699,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.30291253328323364,
-      "rewards/margins": 0.3650719225406647,
-      "rewards/rejected": -0.06215935945510864,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9522091150283813,
+      "logits/rejected": -1.9513927698135376,
+      "logps/chosen": -32.85493469238281,
+      "logps/rejected": -32.53038787841797,
+      "loss": 0.818,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.170864075422287,
+      "rewards/margins": 0.19696101546287537,
+      "rewards/rejected": -0.02609694004058838,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 44.75,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -2.07902193069458,
-      "logits/rejected": -2.085242748260498,
-      "logps/chosen": -31.320343017578125,
-      "logps/rejected": -32.6987419128418,
-      "loss": 0.3034,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.22761543095111847,
-      "rewards/margins": 0.2550276219844818,
-      "rewards/rejected": -0.027412226423621178,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9075958728790283,
+      "logits/rejected": -1.9178472757339478,
+      "logps/chosen": -31.86941909790039,
+      "logps/rejected": -35.343692779541016,
+      "loss": 0.8461,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1341095268726349,
+      "rewards/margins": 0.1560816764831543,
+      "rewards/rejected": -0.021972158923745155,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 94.0,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -2.0127124786376953,
-      "logits/rejected": -2.0121982097625732,
-      "logps/chosen": -33.03043746948242,
-      "logps/rejected": -36.05421447753906,
-      "loss": 0.294,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.3730550706386566,
-      "rewards/margins": 0.3770460784435272,
-      "rewards/rejected": -0.003990952856838703,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.047290325164795,
+      "logits/rejected": -2.0408453941345215,
+      "logps/chosen": -33.33329772949219,
+      "logps/rejected": -29.256567001342773,
+      "loss": 0.8605,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12746797502040863,
+      "rewards/margins": 0.1395258903503418,
+      "rewards/rejected": -0.012057906948029995,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 27.25,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -2.0482752323150635,
-      "logits/rejected": -2.043614625930786,
-      "logps/chosen": -30.266733169555664,
-      "logps/rejected": -29.273147583007812,
-      "loss": 0.3277,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3994811177253723,
-      "rewards/margins": 0.45281118154525757,
-      "rewards/rejected": -0.05333010479807854,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9072849750518799,
+      "logits/rejected": -1.909480333328247,
+      "logps/chosen": -33.89991760253906,
+      "logps/rejected": -31.013565063476562,
+      "loss": 0.8133,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.13565407693386078,
+      "rewards/margins": 0.1920291632413864,
+      "rewards/rejected": -0.05637507513165474,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "grad_norm": 48.75,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.8216171264648438,
-      "logits/rejected": -1.8280115127563477,
-      "logps/chosen": -32.05220031738281,
-      "logps/rejected": -36.293907165527344,
-      "loss": 0.4405,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.30016615986824036,
-      "rewards/margins": 0.45822396874427795,
-      "rewards/rejected": -0.1580578237771988,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "grad_norm": 60.75,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.9820178747177124,
-      "logits/rejected": -1.9757049083709717,
-      "logps/chosen": -33.380653381347656,
-      "logps/rejected": -32.34453582763672,
-      "loss": 0.4084,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.4068370759487152,
-      "rewards/margins": 0.43893057107925415,
-      "rewards/rejected": -0.032093439251184464,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.244844436645508,
-      "eval_logits/rejected": -2.2399790287017822,
-      "eval_logps/chosen": -33.95623016357422,
-      "eval_logps/rejected": -37.43685531616211,
-      "eval_loss": 0.40588921308517456,
-      "eval_rewards/accuracies": 0.5066444873809814,
-      "eval_rewards/chosen": 0.07048822194337845,
-      "eval_rewards/margins": -0.001299927243962884,
-      "eval_rewards/rejected": 0.07178815454244614,
-      "eval_runtime": 145.5279,
-      "eval_samples_per_second": 2.357,
-      "eval_steps_per_second": 0.295,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "grad_norm": 37.5,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -1.981856107711792,
-      "logits/rejected": -1.9890705347061157,
-      "logps/chosen": -28.692569732666016,
-      "logps/rejected": -29.251338958740234,
-      "loss": 0.3376,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3850264847278595,
-      "rewards/margins": 0.5228284597396851,
-      "rewards/rejected": -0.13780196011066437,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "grad_norm": 46.5,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -2.0013296604156494,
-      "logits/rejected": -2.000218152999878,
-      "logps/chosen": -33.88956832885742,
-      "logps/rejected": -30.98602294921875,
-      "loss": 0.244,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.32502850890159607,
-      "rewards/margins": 0.36370736360549927,
-      "rewards/rejected": -0.038678836077451706,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "grad_norm": 74.5,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.9944326877593994,
-      "logits/rejected": -1.9771124124526978,
-      "logps/chosen": -30.677692413330078,
-      "logps/rejected": -33.02484893798828,
-      "loss": 0.2988,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.269562691450119,
-      "rewards/margins": 0.35826626420021057,
-      "rewards/rejected": -0.08870354294776917,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "grad_norm": 38.5,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -2.0252749919891357,
-      "logits/rejected": -2.0271475315093994,
-      "logps/chosen": -36.258460998535156,
-      "logps/rejected": -34.926544189453125,
-      "loss": 0.292,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.31851616501808167,
-      "rewards/margins": 0.3784351646900177,
-      "rewards/rejected": -0.059918999671936035,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "grad_norm": 37.5,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -2.1034398078918457,
-      "logits/rejected": -2.1033284664154053,
-      "logps/chosen": -32.266780853271484,
-      "logps/rejected": -33.54703140258789,
-      "loss": 0.5497,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3299091160297394,
-      "rewards/margins": 0.31813210248947144,
-      "rewards/rejected": 0.011777004227042198,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "grad_norm": 33.75,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -2.0468077659606934,
-      "logits/rejected": -2.0393521785736084,
-      "logps/chosen": -32.022674560546875,
-      "logps/rejected": -36.174320220947266,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.34495481848716736,
-      "rewards/margins": 0.4532076418399811,
-      "rewards/rejected": -0.10825282335281372,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "grad_norm": 59.25,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.9485485553741455,
-      "logits/rejected": -1.944986343383789,
-      "logps/chosen": -33.546749114990234,
-      "logps/rejected": -33.010704040527344,
-      "loss": 0.283,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.34699487686157227,
-      "rewards/margins": 0.4251338541507721,
-      "rewards/rejected": -0.07813899219036102,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 61.25,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -2.071899652481079,
-      "logits/rejected": -2.064889907836914,
-      "logps/chosen": -30.4763240814209,
-      "logps/rejected": -32.066490173339844,
-      "loss": 0.2764,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.316992849111557,
-      "rewards/margins": 0.3826924264431,
-      "rewards/rejected": -0.06569959223270416,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "grad_norm": 69.0,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -2.01814341545105,
-      "logits/rejected": -2.01973557472229,
-      "logps/chosen": -34.93284225463867,
-      "logps/rejected": -33.683841705322266,
-      "loss": 0.2944,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.37830987572669983,
-      "rewards/margins": 0.4164048731327057,
-      "rewards/rejected": -0.038094960153102875,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 44.0,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.9071893692016602,
-      "logits/rejected": -1.9048168659210205,
-      "logps/chosen": -32.796241760253906,
-      "logps/rejected": -36.574031829833984,
-      "loss": 0.2788,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.33844107389450073,
-      "rewards/margins": 0.4371793270111084,
-      "rewards/rejected": -0.09873829782009125,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.2405266761779785,
-      "eval_logits/rejected": -2.2356326580047607,
-      "eval_logps/chosen": -33.95664978027344,
-      "eval_logps/rejected": -37.45263671875,
-      "eval_loss": 0.38657647371292114,
-      "eval_rewards/accuracies": 0.5191029906272888,
-      "eval_rewards/chosen": 0.07010918855667114,
-      "eval_rewards/margins": 0.012523897923529148,
-      "eval_rewards/rejected": 0.05758529528975487,
-      "eval_runtime": 145.2533,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "grad_norm": 37.5,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -2.013305187225342,
-      "logits/rejected": -2.018036127090454,
-      "logps/chosen": -33.06889724731445,
-      "logps/rejected": -31.70903968811035,
-      "loss": 0.3894,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.34961196780204773,
-      "rewards/margins": 0.40624913573265076,
-      "rewards/rejected": -0.05663713067770004,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "grad_norm": 43.5,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -2.0473179817199707,
-      "logits/rejected": -2.0582363605499268,
-      "logps/chosen": -30.8958797454834,
-      "logps/rejected": -31.604564666748047,
-      "loss": 0.359,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.44507941603660583,
-      "rewards/margins": 0.5322669744491577,
-      "rewards/rejected": -0.08718754351139069,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "grad_norm": 49.5,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.974761962890625,
-      "logits/rejected": -1.9713010787963867,
-      "logps/chosen": -32.03337097167969,
-      "logps/rejected": -32.46263885498047,
-      "loss": 0.2845,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.34842735528945923,
-      "rewards/margins": 0.4383048117160797,
-      "rewards/rejected": -0.08987746387720108,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 56.0,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.8440120220184326,
-      "logits/rejected": -1.8531720638275146,
-      "logps/chosen": -32.23934555053711,
-      "logps/rejected": -31.933460235595703,
-      "loss": 0.2635,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.29704010486602783,
-      "rewards/margins": 0.36676424741744995,
-      "rewards/rejected": -0.06972410529851913,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "grad_norm": 41.75,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.9423996210098267,
-      "logits/rejected": -1.9359699487686157,
-      "logps/chosen": -33.36035919189453,
-      "logps/rejected": -31.000152587890625,
-      "loss": 0.3772,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.3131468892097473,
-      "rewards/margins": 0.32107049226760864,
-      "rewards/rejected": -0.007923613302409649,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "grad_norm": 147.0,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -2.0321927070617676,
-      "logits/rejected": -2.030066967010498,
-      "logps/chosen": -34.12587356567383,
-      "logps/rejected": -31.518299102783203,
-      "loss": 0.5612,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.40079349279403687,
-      "rewards/margins": 0.41961637139320374,
-      "rewards/rejected": -0.01882282830774784,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "grad_norm": 28.125,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.8660882711410522,
-      "logits/rejected": -1.863891363143921,
-      "logps/chosen": -32.46380615234375,
-      "logps/rejected": -31.035659790039062,
-      "loss": 0.3042,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.2993435263633728,
-      "rewards/margins": 0.3293627202510834,
-      "rewards/rejected": -0.030019233003258705,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "grad_norm": 126.0,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -2.0015358924865723,
-      "logits/rejected": -1.998284935951233,
-      "logps/chosen": -30.789913177490234,
-      "logps/rejected": -34.707244873046875,
-      "loss": 0.3323,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.2832598090171814,
-      "rewards/margins": 0.3562392592430115,
-      "rewards/rejected": -0.07297946512699127,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "grad_norm": 26.125,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.946082353591919,
-      "logits/rejected": -1.9497697353363037,
-      "logps/chosen": -29.2518310546875,
-      "logps/rejected": -30.736743927001953,
-      "loss": 0.3063,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3268837332725525,
-      "rewards/margins": 0.3969722092151642,
-      "rewards/rejected": -0.07008843123912811,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "grad_norm": 46.25,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.9876987934112549,
-      "logits/rejected": -1.9874299764633179,
-      "logps/chosen": -31.799388885498047,
-      "logps/rejected": -31.247623443603516,
-      "loss": 0.3874,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3799284100532532,
-      "rewards/margins": 0.4290773868560791,
-      "rewards/rejected": -0.049148980528116226,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.2469940185546875,
-      "eval_logits/rejected": -2.2421164512634277,
-      "eval_logps/chosen": -33.955604553222656,
-      "eval_logps/rejected": -37.417686462402344,
-      "eval_loss": 0.426466703414917,
-      "eval_rewards/accuracies": 0.472591370344162,
-      "eval_rewards/chosen": 0.07105044275522232,
-      "eval_rewards/margins": -0.017991825938224792,
-      "eval_rewards/rejected": 0.08904226124286652,
-      "eval_runtime": 145.3878,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "grad_norm": 23.5,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -2.0506019592285156,
-      "logits/rejected": -2.051326274871826,
-      "logps/chosen": -34.585548400878906,
-      "logps/rejected": -33.11695098876953,
-      "loss": 0.2877,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.4001394212245941,
-      "rewards/margins": 0.4296254515647888,
-      "rewards/rejected": -0.02948596142232418,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "grad_norm": 69.5,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -2.013078212738037,
-      "logits/rejected": -2.010453701019287,
-      "logps/chosen": -32.696006774902344,
-      "logps/rejected": -32.14519500732422,
-      "loss": 0.3336,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.3864508867263794,
-      "rewards/margins": 0.4542017877101898,
-      "rewards/rejected": -0.06775090843439102,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "grad_norm": 31.875,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -2.02937650680542,
-      "logits/rejected": -2.0202343463897705,
-      "logps/chosen": -32.354148864746094,
-      "logps/rejected": -30.961654663085938,
-      "loss": 0.3879,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.3797711730003357,
-      "rewards/margins": 0.40343213081359863,
-      "rewards/rejected": -0.023660968989133835,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "grad_norm": 119.0,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -2.035832405090332,
-      "logits/rejected": -2.032696485519409,
-      "logps/chosen": -31.714279174804688,
-      "logps/rejected": -32.49519348144531,
-      "loss": 0.3205,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.43572768568992615,
-      "rewards/margins": 0.5039243102073669,
-      "rewards/rejected": -0.06819657981395721,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "grad_norm": 24.375,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -2.0391454696655273,
-      "logits/rejected": -2.041524887084961,
-      "logps/chosen": -33.29151916503906,
-      "logps/rejected": -33.737464904785156,
-      "loss": 0.2569,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.32596832513809204,
-      "rewards/margins": 0.37462925910949707,
-      "rewards/rejected": -0.04866095632314682,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "grad_norm": 32.5,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.9535996913909912,
-      "logits/rejected": -1.9540789127349854,
-      "logps/chosen": -33.57703399658203,
-      "logps/rejected": -31.611495971679688,
-      "loss": 0.2509,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.33281606435775757,
-      "rewards/margins": 0.3366300165653229,
-      "rewards/rejected": -0.0038140162359923124,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "grad_norm": 46.75,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.9932886362075806,
-      "logits/rejected": -1.983457326889038,
-      "logps/chosen": -33.600196838378906,
-      "logps/rejected": -32.383522033691406,
-      "loss": 0.2964,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.36097538471221924,
-      "rewards/margins": 0.4291335642337799,
-      "rewards/rejected": -0.06815823167562485,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "grad_norm": 53.25,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.8898265361785889,
-      "logits/rejected": -1.8866018056869507,
-      "logps/chosen": -31.05093002319336,
-      "logps/rejected": -34.363487243652344,
-      "loss": 0.2359,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.36902502179145813,
-      "rewards/margins": 0.4094383120536804,
-      "rewards/rejected": -0.0404132641851902,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "grad_norm": 99.5,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.9469459056854248,
-      "logits/rejected": -1.9461286067962646,
-      "logps/chosen": -34.85956954956055,
-      "logps/rejected": -35.397823333740234,
-      "loss": 0.313,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.36964648962020874,
-      "rewards/margins": 0.4092490077018738,
-      "rewards/rejected": -0.03960253670811653,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "grad_norm": 60.0,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.9282480478286743,
-      "logits/rejected": -1.9313417673110962,
-      "logps/chosen": -32.19511032104492,
-      "logps/rejected": -35.74637985229492,
-      "loss": 0.2695,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.3669659197330475,
-      "rewards/margins": 0.46120119094848633,
-      "rewards/rejected": -0.09423529356718063,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.247775077819824,
-      "eval_logits/rejected": -2.2429134845733643,
-      "eval_logps/chosen": -33.943904876708984,
-      "eval_logps/rejected": -37.41926193237305,
-      "eval_loss": 0.402832955121994,
-      "eval_rewards/accuracies": 0.5078904032707214,
-      "eval_rewards/chosen": 0.08158490061759949,
-      "eval_rewards/margins": -0.006037927698343992,
-      "eval_rewards/rejected": 0.08762282878160477,
-      "eval_runtime": 145.2777,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "grad_norm": 51.75,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -2.040821075439453,
-      "logits/rejected": -2.0436055660247803,
-      "logps/chosen": -34.49801254272461,
-      "logps/rejected": -34.32398223876953,
-      "loss": 0.3608,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.35781916975975037,
-      "rewards/margins": 0.40626224875450134,
-      "rewards/rejected": -0.04844306781888008,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "grad_norm": 110.5,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.8965297937393188,
-      "logits/rejected": -1.8905613422393799,
-      "logps/chosen": -34.4133415222168,
-      "logps/rejected": -32.38819885253906,
-      "loss": 0.2806,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.3779004216194153,
-      "rewards/margins": 0.4106220602989197,
-      "rewards/rejected": -0.03272164613008499,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 39.5,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -2.0699164867401123,
-      "logits/rejected": -2.0668702125549316,
-      "logps/chosen": -30.270557403564453,
-      "logps/rejected": -31.52914047241211,
-      "loss": 0.2811,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.3441884219646454,
-      "rewards/margins": 0.41703981161117554,
-      "rewards/rejected": -0.07285144180059433,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "grad_norm": 63.0,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.9103057384490967,
-      "logits/rejected": -1.910112738609314,
-      "logps/chosen": -33.07831573486328,
-      "logps/rejected": -37.679222106933594,
-      "loss": 0.2643,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.3463733494281769,
-      "rewards/margins": 0.39264675974845886,
-      "rewards/rejected": -0.04627341777086258,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "grad_norm": 60.75,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.7918742895126343,
-      "logits/rejected": -1.7970874309539795,
-      "logps/chosen": -36.49778366088867,
-      "logps/rejected": -34.00720977783203,
-      "loss": 0.2697,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.3912108838558197,
-      "rewards/margins": 0.4225967526435852,
-      "rewards/rejected": -0.031385865062475204,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "grad_norm": 51.25,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -2.0016860961914062,
-      "logits/rejected": -2.003040313720703,
-      "logps/chosen": -32.25410461425781,
-      "logps/rejected": -32.861629486083984,
-      "loss": 0.337,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.31641632318496704,
-      "rewards/margins": 0.3905896544456482,
-      "rewards/rejected": -0.07417333871126175,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 67.0,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.9714769124984741,
-      "logits/rejected": -1.9706627130508423,
-      "logps/chosen": -31.806903839111328,
-      "logps/rejected": -34.992042541503906,
-      "loss": 0.3126,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3318381905555725,
-      "rewards/margins": 0.29933661222457886,
-      "rewards/rejected": 0.03250158578157425,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "grad_norm": 43.25,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.971329689025879,
-      "logits/rejected": -1.9705543518066406,
-      "logps/chosen": -34.23518371582031,
-      "logps/rejected": -31.475284576416016,
-      "loss": 0.2613,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.371949702501297,
-      "rewards/margins": 0.42213401198387146,
-      "rewards/rejected": -0.05018440634012222,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "grad_norm": 41.0,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -2.003086566925049,
-      "logits/rejected": -2.0015482902526855,
-      "logps/chosen": -29.147836685180664,
-      "logps/rejected": -32.72181701660156,
-      "loss": 0.1641,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.37381279468536377,
-      "rewards/margins": 0.450767457485199,
-      "rewards/rejected": -0.07695472240447998,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "grad_norm": 28.375,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -2.0545146465301514,
-      "logits/rejected": -2.0508596897125244,
-      "logps/chosen": -31.972219467163086,
-      "logps/rejected": -34.25817108154297,
-      "loss": 0.1725,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.4094924330711365,
-      "rewards/margins": 0.46271318197250366,
-      "rewards/rejected": -0.05322074890136719,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.246337890625,
-      "eval_logits/rejected": -2.2414615154266357,
-      "eval_logps/chosen": -33.92709732055664,
-      "eval_logps/rejected": -37.39698791503906,
-      "eval_loss": 0.4082644581794739,
-      "eval_rewards/accuracies": 0.4821428656578064,
-      "eval_rewards/chosen": 0.09670902043581009,
-      "eval_rewards/margins": -0.010959767736494541,
-      "eval_rewards/rejected": 0.10766878724098206,
-      "eval_runtime": 145.3536,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "grad_norm": 25.625,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.8916804790496826,
-      "logits/rejected": -1.8837049007415771,
-      "logps/chosen": -34.002410888671875,
-      "logps/rejected": -34.613807678222656,
-      "loss": 0.2292,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.4084460735321045,
-      "rewards/margins": 0.4567646086215973,
-      "rewards/rejected": -0.04831857234239578,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "grad_norm": 42.0,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.967140555381775,
-      "logits/rejected": -1.9777908325195312,
-      "logps/chosen": -35.31997299194336,
-      "logps/rejected": -32.446533203125,
-      "loss": 0.358,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.33814600110054016,
-      "rewards/margins": 0.35173219442367554,
-      "rewards/rejected": -0.013586151413619518,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "grad_norm": 52.0,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -2.01682710647583,
-      "logits/rejected": -2.0210189819335938,
-      "logps/chosen": -34.396095275878906,
-      "logps/rejected": -29.328195571899414,
-      "loss": 0.1479,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4560183584690094,
-      "rewards/margins": 0.5099755525588989,
-      "rewards/rejected": -0.05395715311169624,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "grad_norm": 41.25,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -2.016162157058716,
-      "logits/rejected": -2.0238842964172363,
-      "logps/chosen": -34.38965606689453,
-      "logps/rejected": -29.9200439453125,
-      "loss": 0.2734,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3585363030433655,
-      "rewards/margins": 0.42364779114723206,
-      "rewards/rejected": -0.06511145085096359,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "grad_norm": 51.0,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.9484409093856812,
-      "logits/rejected": -1.9388576745986938,
-      "logps/chosen": -31.393484115600586,
-      "logps/rejected": -32.06694030761719,
-      "loss": 0.1541,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.35973984003067017,
-      "rewards/margins": 0.40756136178970337,
-      "rewards/rejected": -0.04782148450613022,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "grad_norm": 28.75,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -2.0893542766571045,
-      "logits/rejected": -2.079577922821045,
-      "logps/chosen": -25.801733016967773,
-      "logps/rejected": -29.940242767333984,
-      "loss": 0.1453,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.38010627031326294,
-      "rewards/margins": 0.44122153520584106,
-      "rewards/rejected": -0.06111522391438484,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "grad_norm": 63.5,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.9405052661895752,
-      "logits/rejected": -1.9417035579681396,
-      "logps/chosen": -33.93370819091797,
-      "logps/rejected": -30.114543914794922,
-      "loss": 0.2379,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.35936418175697327,
-      "rewards/margins": 0.478200763463974,
-      "rewards/rejected": -0.11883661895990372,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "grad_norm": 29.25,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.9570789337158203,
-      "logits/rejected": -1.957831621170044,
-      "logps/chosen": -31.962173461914062,
-      "logps/rejected": -33.07716751098633,
-      "loss": 0.2589,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.3843681216239929,
-      "rewards/margins": 0.40799227356910706,
-      "rewards/rejected": -0.023624107241630554,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "grad_norm": 30.625,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.860047698020935,
-      "logits/rejected": -1.858214020729065,
-      "logps/chosen": -35.269798278808594,
-      "logps/rejected": -35.90485382080078,
-      "loss": 0.2443,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.3556040823459625,
-      "rewards/margins": 0.3804590404033661,
-      "rewards/rejected": -0.024854950606822968,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "grad_norm": 104.5,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -2.0041489601135254,
-      "logits/rejected": -1.9973175525665283,
-      "logps/chosen": -31.497005462646484,
-      "logps/rejected": -32.764808654785156,
-      "loss": 0.2502,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.45506516098976135,
-      "rewards/margins": 0.46113377809524536,
-      "rewards/rejected": -0.006068611983209848,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.248673439025879,
-      "eval_logits/rejected": -2.2437942028045654,
-      "eval_logps/chosen": -33.90636444091797,
-      "eval_logps/rejected": -37.37092971801758,
-      "eval_loss": 0.40991199016571045,
-      "eval_rewards/accuracies": 0.49003320932388306,
-      "eval_rewards/chosen": 0.11537089198827744,
-      "eval_rewards/margins": -0.015749912708997726,
-      "eval_rewards/rejected": 0.13112080097198486,
-      "eval_runtime": 145.395,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "grad_norm": 15.8125,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -2.0507731437683105,
-      "logits/rejected": -2.0459985733032227,
-      "logps/chosen": -33.123680114746094,
-      "logps/rejected": -34.560203552246094,
-      "loss": 0.1461,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.39060264825820923,
-      "rewards/margins": 0.4190269410610199,
-      "rewards/rejected": -0.028424326330423355,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "grad_norm": 57.25,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.9670826196670532,
-      "logits/rejected": -1.9720354080200195,
-      "logps/chosen": -34.90446090698242,
-      "logps/rejected": -32.519283294677734,
-      "loss": 0.2881,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.3997105062007904,
-      "rewards/margins": 0.4171512722969055,
-      "rewards/rejected": -0.01744077168405056,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "grad_norm": 26.375,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -2.0626909732818604,
-      "logits/rejected": -2.053279161453247,
-      "logps/chosen": -31.813243865966797,
-      "logps/rejected": -34.46527099609375,
-      "loss": 0.1157,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.4245172142982483,
-      "rewards/margins": 0.44474273920059204,
-      "rewards/rejected": -0.020225506275892258,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "grad_norm": 85.0,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.98918879032135,
-      "logits/rejected": -1.986096739768982,
-      "logps/chosen": -30.814416885375977,
-      "logps/rejected": -32.54015350341797,
-      "loss": 0.2202,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.4266343116760254,
-      "rewards/margins": 0.49393558502197266,
-      "rewards/rejected": -0.06730121374130249,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "grad_norm": 15.1875,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.932186484336853,
-      "logits/rejected": -1.9396946430206299,
-      "logps/chosen": -32.45960235595703,
-      "logps/rejected": -35.62566375732422,
-      "loss": 0.1934,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3703078627586365,
-      "rewards/margins": 0.43846622109413147,
-      "rewards/rejected": -0.06815837323665619,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "grad_norm": 73.5,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.940987229347229,
-      "logits/rejected": -1.934152603149414,
-      "logps/chosen": -33.384063720703125,
-      "logps/rejected": -36.64484405517578,
-      "loss": 0.1978,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.4000038206577301,
-      "rewards/margins": 0.44143620133399963,
-      "rewards/rejected": -0.041432395577430725,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "grad_norm": 18.0,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.942278265953064,
-      "logits/rejected": -1.9429972171783447,
-      "logps/chosen": -30.803075790405273,
-      "logps/rejected": -33.071388244628906,
-      "loss": 0.2326,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.38999220728874207,
-      "rewards/margins": 0.47110748291015625,
-      "rewards/rejected": -0.0811152458190918,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "grad_norm": 31.75,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.946674108505249,
-      "logits/rejected": -1.9579198360443115,
-      "logps/chosen": -32.60912322998047,
-      "logps/rejected": -34.126827239990234,
-      "loss": 0.264,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.4208148419857025,
-      "rewards/margins": 0.4168703556060791,
-      "rewards/rejected": 0.003944486379623413,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "grad_norm": 42.0,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -2.0197396278381348,
-      "logits/rejected": -2.017103672027588,
-      "logps/chosen": -35.6496696472168,
-      "logps/rejected": -32.41356658935547,
-      "loss": 0.2859,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.4242428243160248,
-      "rewards/margins": 0.4371464252471924,
-      "rewards/rejected": -0.012903605587780476,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "grad_norm": 35.25,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.9874387979507446,
-      "logits/rejected": -1.9911830425262451,
-      "logps/chosen": -34.0231819152832,
-      "logps/rejected": -34.56896209716797,
-      "loss": 0.1529,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.39610037207603455,
-      "rewards/margins": 0.47072356939315796,
-      "rewards/rejected": -0.07462319731712341,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.2477176189422607,
-      "eval_logits/rejected": -2.242847442626953,
-      "eval_logps/chosen": -33.89875411987305,
-      "eval_logps/rejected": -37.377010345458984,
-      "eval_loss": 0.3879217505455017,
-      "eval_rewards/accuracies": 0.5215947031974792,
-      "eval_rewards/chosen": 0.12221997231245041,
-      "eval_rewards/margins": -0.003430851735174656,
-      "eval_rewards/rejected": 0.1256508231163025,
-      "eval_runtime": 145.3271,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "grad_norm": 40.25,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.9966695308685303,
-      "logits/rejected": -2.0017590522766113,
-      "logps/chosen": -32.33439636230469,
-      "logps/rejected": -32.539424896240234,
-      "loss": 0.1576,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.41942542791366577,
-      "rewards/margins": 0.43577662110328674,
-      "rewards/rejected": -0.01635116897523403,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "grad_norm": 53.25,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.959848165512085,
-      "logits/rejected": -1.962561011314392,
-      "logps/chosen": -30.024948120117188,
-      "logps/rejected": -31.70804786682129,
-      "loss": 0.2273,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.3786233961582184,
-      "rewards/margins": 0.3866207003593445,
-      "rewards/rejected": -0.007997283712029457,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "grad_norm": 23.125,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -2.017338514328003,
-      "logits/rejected": -2.0114893913269043,
-      "logps/chosen": -33.496673583984375,
-      "logps/rejected": -33.55792999267578,
-      "loss": 0.1431,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4431969225406647,
-      "rewards/margins": 0.4791356027126312,
-      "rewards/rejected": -0.035938672721385956,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "grad_norm": 56.0,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.9453651905059814,
-      "logits/rejected": -1.9490783214569092,
-      "logps/chosen": -35.259788513183594,
-      "logps/rejected": -34.503028869628906,
-      "loss": 0.2102,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4445478916168213,
-      "rewards/margins": 0.519649863243103,
-      "rewards/rejected": -0.07510194182395935,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "grad_norm": 35.75,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.9455549716949463,
-      "logits/rejected": -1.95868718624115,
-      "logps/chosen": -31.420841217041016,
-      "logps/rejected": -33.076026916503906,
-      "loss": 0.2334,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.39784207940101624,
-      "rewards/margins": 0.4482339024543762,
-      "rewards/rejected": -0.05039183050394058,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "grad_norm": 58.75,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.9375944137573242,
-      "logits/rejected": -1.9310089349746704,
-      "logps/chosen": -31.681589126586914,
-      "logps/rejected": -33.36168670654297,
-      "loss": 0.1253,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.42997032403945923,
-      "rewards/margins": 0.46931910514831543,
-      "rewards/rejected": -0.03934880718588829,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "grad_norm": 40.5,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -2.0641446113586426,
-      "logits/rejected": -2.0649874210357666,
-      "logps/chosen": -33.09912109375,
-      "logps/rejected": -32.90647888183594,
-      "loss": 0.1818,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.4031626284122467,
-      "rewards/margins": 0.4587678909301758,
-      "rewards/rejected": -0.05560523271560669,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "grad_norm": 25.25,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.9966720342636108,
-      "logits/rejected": -1.9955018758773804,
-      "logps/chosen": -33.611534118652344,
-      "logps/rejected": -33.192691802978516,
-      "loss": 0.1975,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.43046802282333374,
-      "rewards/margins": 0.431241899728775,
-      "rewards/rejected": -0.0007738359272480011,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "grad_norm": 24.625,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -2.0245463848114014,
-      "logits/rejected": -2.0170249938964844,
-      "logps/chosen": -33.81514358520508,
-      "logps/rejected": -31.061843872070312,
-      "loss": 0.1443,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.3696584105491638,
-      "rewards/margins": 0.387938916683197,
-      "rewards/rejected": -0.018280521035194397,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "grad_norm": 88.0,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.9909054040908813,
-      "logits/rejected": -1.9901247024536133,
-      "logps/chosen": -34.939388275146484,
-      "logps/rejected": -36.37110137939453,
-      "loss": 0.1583,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.43112340569496155,
-      "rewards/margins": 0.44767728447914124,
-      "rewards/rejected": -0.01655389368534088,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.2481679916381836,
-      "eval_logits/rejected": -2.243297576904297,
-      "eval_logps/chosen": -33.90199279785156,
-      "eval_logps/rejected": -37.37769317626953,
-      "eval_loss": 0.3968319892883301,
-      "eval_rewards/accuracies": 0.4875415563583374,
-      "eval_rewards/chosen": 0.11930261552333832,
-      "eval_rewards/margins": -0.0057318080216646194,
-      "eval_rewards/rejected": 0.1250344216823578,
-      "eval_runtime": 145.2343,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "grad_norm": 34.0,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -2.0373494625091553,
-      "logits/rejected": -2.0427050590515137,
-      "logps/chosen": -35.03519821166992,
-      "logps/rejected": -35.56550216674805,
-      "loss": 0.1969,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.38872379064559937,
-      "rewards/margins": 0.40845227241516113,
-      "rewards/rejected": -0.01972847804427147,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "grad_norm": 91.0,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -2.009969472885132,
-      "logits/rejected": -2.0080225467681885,
-      "logps/chosen": -32.2147216796875,
-      "logps/rejected": -32.992332458496094,
-      "loss": 0.2049,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.36133700609207153,
-      "rewards/margins": 0.36995649337768555,
-      "rewards/rejected": -0.008619492873549461,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "grad_norm": 42.0,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.9126440286636353,
-      "logits/rejected": -1.9155027866363525,
-      "logps/chosen": -29.281444549560547,
-      "logps/rejected": -30.237567901611328,
-      "loss": 0.18,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.42047175765037537,
-      "rewards/margins": 0.4137454926967621,
-      "rewards/rejected": 0.006726269610226154,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "grad_norm": 33.0,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -2.009428024291992,
-      "logits/rejected": -2.0063159465789795,
-      "logps/chosen": -32.74279022216797,
-      "logps/rejected": -32.62069320678711,
-      "loss": 0.1567,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.4148639142513275,
-      "rewards/margins": 0.49929580092430115,
-      "rewards/rejected": -0.08443190902471542,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "grad_norm": 50.25,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.931823968887329,
-      "logits/rejected": -1.925490140914917,
-      "logps/chosen": -27.730154037475586,
-      "logps/rejected": -32.26335906982422,
-      "loss": 0.1296,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3670499324798584,
-      "rewards/margins": 0.47060221433639526,
-      "rewards/rejected": -0.10355228185653687,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "grad_norm": 35.25,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -2.0501132011413574,
-      "logits/rejected": -2.047624111175537,
-      "logps/chosen": -32.637855529785156,
-      "logps/rejected": -35.0833625793457,
-      "loss": 0.0915,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.41239070892333984,
-      "rewards/margins": 0.4050343632698059,
-      "rewards/rejected": 0.007356312125921249,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "grad_norm": 17.625,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -2.0300636291503906,
-      "logits/rejected": -2.0281882286071777,
-      "logps/chosen": -31.161453247070312,
-      "logps/rejected": -32.45496368408203,
-      "loss": 0.0941,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4076564908027649,
-      "rewards/margins": 0.4894997179508209,
-      "rewards/rejected": -0.08184324204921722,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "grad_norm": 32.25,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -2.02311110496521,
-      "logits/rejected": -2.0107522010803223,
-      "logps/chosen": -35.538490295410156,
-      "logps/rejected": -34.12282180786133,
-      "loss": 0.1417,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4561285972595215,
-      "rewards/margins": 0.5023427605628967,
-      "rewards/rejected": -0.046214111149311066,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "grad_norm": 55.0,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.9117752313613892,
-      "logits/rejected": -1.9064388275146484,
-      "logps/chosen": -34.895999908447266,
-      "logps/rejected": -34.239891052246094,
-      "loss": 0.1368,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.4083231985569,
-      "rewards/margins": 0.41019004583358765,
-      "rewards/rejected": -0.001866865553893149,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "grad_norm": 54.5,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.9550445079803467,
-      "logits/rejected": -1.958034873008728,
-      "logps/chosen": -32.1730842590332,
-      "logps/rejected": -31.49075698852539,
-      "loss": 0.113,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.39780890941619873,
-      "rewards/margins": 0.4019277095794678,
-      "rewards/rejected": -0.004118794109672308,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.2470004558563232,
-      "eval_logits/rejected": -2.2421324253082275,
-      "eval_logps/chosen": -33.908164978027344,
-      "eval_logps/rejected": -37.38742446899414,
-      "eval_loss": 0.3848971724510193,
-      "eval_rewards/accuracies": 0.47840529680252075,
-      "eval_rewards/chosen": 0.11374907940626144,
-      "eval_rewards/margins": -0.002530111465603113,
-      "eval_rewards/rejected": 0.11627919226884842,
-      "eval_runtime": 145.1536,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "grad_norm": 23.875,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.9873783588409424,
-      "logits/rejected": -1.9827861785888672,
-      "logps/chosen": -30.8137264251709,
-      "logps/rejected": -30.97092628479004,
-      "loss": 0.1001,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.41394877433776855,
-      "rewards/margins": 0.4704950451850891,
-      "rewards/rejected": -0.05654625967144966,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "grad_norm": 18.875,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.9986066818237305,
-      "logits/rejected": -1.9930963516235352,
-      "logps/chosen": -34.82990264892578,
-      "logps/rejected": -34.601905822753906,
-      "loss": 0.1136,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.399636447429657,
-      "rewards/margins": 0.4284568727016449,
-      "rewards/rejected": -0.02882050909101963,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "grad_norm": 35.0,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -2.000286102294922,
-      "logits/rejected": -1.9870882034301758,
-      "logps/chosen": -32.6419677734375,
-      "logps/rejected": -34.18825149536133,
-      "loss": 0.1139,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.4068435728549957,
-      "rewards/margins": 0.4684053957462311,
-      "rewards/rejected": -0.06156182289123535,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "grad_norm": 29.0,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -2.044322967529297,
-      "logits/rejected": -2.0414676666259766,
-      "logps/chosen": -34.62710189819336,
-      "logps/rejected": -31.878192901611328,
-      "loss": 0.1302,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.36521658301353455,
-      "rewards/margins": 0.399711012840271,
-      "rewards/rejected": -0.03449442610144615,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "grad_norm": 51.5,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.9766407012939453,
-      "logits/rejected": -1.9737846851348877,
-      "logps/chosen": -33.98097610473633,
-      "logps/rejected": -34.84381103515625,
-      "loss": 0.1699,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.37366223335266113,
-      "rewards/margins": 0.4156786799430847,
-      "rewards/rejected": -0.04201648384332657,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "grad_norm": 30.75,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -2.038552761077881,
-      "logits/rejected": -2.0333991050720215,
-      "logps/chosen": -30.33209800720215,
-      "logps/rejected": -32.44548797607422,
-      "loss": 0.0978,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3858305513858795,
-      "rewards/margins": 0.44247692823410034,
-      "rewards/rejected": -0.05664638429880142,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "grad_norm": 28.375,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -2.065164089202881,
-      "logits/rejected": -2.069824457168579,
-      "logps/chosen": -32.27547073364258,
-      "logps/rejected": -31.700992584228516,
-      "loss": 0.1001,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3919737637042999,
-      "rewards/margins": 0.4096389412879944,
-      "rewards/rejected": -0.017665188759565353,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "grad_norm": 18.625,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.9599748849868774,
-      "logits/rejected": -1.9670900106430054,
-      "logps/chosen": -34.481956481933594,
-      "logps/rejected": -32.67145538330078,
-      "loss": 0.1083,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3827614486217499,
-      "rewards/margins": 0.4486331343650818,
-      "rewards/rejected": -0.06587164849042892,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "grad_norm": 33.25,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.862562894821167,
-      "logits/rejected": -1.864749550819397,
-      "logps/chosen": -32.70376968383789,
-      "logps/rejected": -36.58349609375,
-      "loss": 0.1038,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3853146433830261,
-      "rewards/margins": 0.41162100434303284,
-      "rewards/rejected": -0.026306327432394028,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "grad_norm": 64.0,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.9185287952423096,
-      "logits/rejected": -1.921884298324585,
-      "logps/chosen": -32.8210334777832,
-      "logps/rejected": -32.53430938720703,
-      "loss": 0.0937,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.35607367753982544,
-      "rewards/margins": 0.41920334100723267,
-      "rewards/rejected": -0.06312969326972961,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.2471628189086914,
-      "eval_logits/rejected": -2.2422919273376465,
-      "eval_logps/chosen": -33.897342681884766,
-      "eval_logps/rejected": -37.385860443115234,
-      "eval_loss": 0.37376365065574646,
-      "eval_rewards/accuracies": 0.5045680999755859,
-      "eval_rewards/chosen": 0.1234876811504364,
-      "eval_rewards/margins": 0.005806357134133577,
-      "eval_rewards/rejected": 0.1176813393831253,
-      "eval_runtime": 145.1552,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "grad_norm": 37.5,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -2.027667999267578,
-      "logits/rejected": -2.0075087547302246,
-      "logps/chosen": -30.6841983795166,
-      "logps/rejected": -35.201026916503906,
-      "loss": 0.135,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.38628271222114563,
-      "rewards/margins": 0.4174365997314453,
-      "rewards/rejected": -0.031153928488492966,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "grad_norm": 141.0,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -2.0063624382019043,
-      "logits/rejected": -2.012112855911255,
-      "logps/chosen": -32.609859466552734,
-      "logps/rejected": -32.543277740478516,
-      "loss": 0.1282,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.38153642416000366,
-      "rewards/margins": 0.4288099706172943,
-      "rewards/rejected": -0.04727357253432274,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "grad_norm": 13.625,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.9746071100234985,
-      "logits/rejected": -1.969681739807129,
-      "logps/chosen": -32.591461181640625,
-      "logps/rejected": -34.600608825683594,
-      "loss": 0.1059,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.38793960213661194,
-      "rewards/margins": 0.4134295880794525,
-      "rewards/rejected": -0.025490064173936844,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "grad_norm": 35.75,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.8414928913116455,
-      "logits/rejected": -1.8512153625488281,
-      "logps/chosen": -33.299381256103516,
-      "logps/rejected": -31.048131942749023,
-      "loss": 0.0735,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 0.4501881003379822,
-      "rewards/margins": 0.4628051221370697,
-      "rewards/rejected": -0.012616944499313831,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "grad_norm": 22.5,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.9015308618545532,
-      "logits/rejected": -1.9062535762786865,
-      "logps/chosen": -33.4455680847168,
-      "logps/rejected": -33.258026123046875,
-      "loss": 0.0823,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.4548335671424866,
-      "rewards/margins": 0.5027266144752502,
-      "rewards/rejected": -0.04789309203624725,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "grad_norm": 56.25,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.9281432628631592,
-      "logits/rejected": -1.9337003231048584,
-      "logps/chosen": -33.40667724609375,
-      "logps/rejected": -34.605411529541016,
-      "loss": 0.0912,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.38248568773269653,
-      "rewards/margins": 0.42059844732284546,
-      "rewards/rejected": -0.038112785667181015,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "grad_norm": 11.375,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -2.017787456512451,
-      "logits/rejected": -2.0132956504821777,
-      "logps/chosen": -32.1794319152832,
-      "logps/rejected": -30.92777442932129,
-      "loss": 0.1078,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.4106277525424957,
-      "rewards/margins": 0.4249129891395569,
-      "rewards/rejected": -0.014285224489867687,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "grad_norm": 19.875,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.9410808086395264,
-      "logits/rejected": -1.9427919387817383,
-      "logps/chosen": -29.26833152770996,
-      "logps/rejected": -30.779708862304688,
-      "loss": 0.0842,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.35187023878097534,
-      "rewards/margins": 0.3838461935520172,
-      "rewards/rejected": -0.031975992023944855,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "grad_norm": 16.125,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.926817536354065,
-      "logits/rejected": -1.931318998336792,
-      "logps/chosen": -31.545446395874023,
-      "logps/rejected": -33.034786224365234,
-      "loss": 0.0861,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.4266428053379059,
-      "rewards/margins": 0.46869271993637085,
-      "rewards/rejected": -0.042049963027238846,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "grad_norm": 29.0,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.9985454082489014,
-      "logits/rejected": -2.00184965133667,
-      "logps/chosen": -34.45569610595703,
-      "logps/rejected": -33.49158477783203,
-      "loss": 0.0815,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.4273880422115326,
-      "rewards/margins": 0.4886067509651184,
-      "rewards/rejected": -0.061218757182359695,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.2475900650024414,
-      "eval_logits/rejected": -2.2427213191986084,
-      "eval_logps/chosen": -33.8858757019043,
-      "eval_logps/rejected": -37.38360595703125,
-      "eval_loss": 0.35954806208610535,
-      "eval_rewards/accuracies": 0.5224252939224243,
-      "eval_rewards/chosen": 0.13380877673625946,
-      "eval_rewards/margins": 0.014098312705755234,
-      "eval_rewards/rejected": 0.11971045285463333,
-      "eval_runtime": 145.2937,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "grad_norm": 38.25,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -2.0676581859588623,
-      "logits/rejected": -2.060824155807495,
-      "logps/chosen": -33.353126525878906,
-      "logps/rejected": -34.27102279663086,
-      "loss": 0.1028,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.4044764041900635,
-      "rewards/margins": 0.42828792333602905,
-      "rewards/rejected": -0.023811563849449158,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "grad_norm": 46.0,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.9524263143539429,
-      "logits/rejected": -1.9631872177124023,
-      "logps/chosen": -33.7514533996582,
-      "logps/rejected": -33.68230438232422,
-      "loss": 0.1069,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.3580663800239563,
-      "rewards/margins": 0.44016751646995544,
-      "rewards/rejected": -0.08210111409425735,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "grad_norm": 33.0,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -2.038909912109375,
-      "logits/rejected": -2.0507607460021973,
-      "logps/chosen": -34.61919021606445,
-      "logps/rejected": -33.713623046875,
-      "loss": 0.1328,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.39745327830314636,
-      "rewards/margins": 0.465212881565094,
-      "rewards/rejected": -0.06775961816310883,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "grad_norm": 20.125,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -2.0287575721740723,
-      "logits/rejected": -2.030729055404663,
-      "logps/chosen": -32.74304962158203,
-      "logps/rejected": -33.90821075439453,
-      "loss": 0.1046,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.4188024401664734,
-      "rewards/margins": 0.46222352981567383,
-      "rewards/rejected": -0.04342114180326462,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "grad_norm": 20.875,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -2.010601758956909,
-      "logits/rejected": -2.0033953189849854,
-      "logps/chosen": -35.09027862548828,
-      "logps/rejected": -33.78215789794922,
-      "loss": 0.0945,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.43024691939353943,
-      "rewards/margins": 0.4408265948295593,
-      "rewards/rejected": -0.010579647496342659,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "grad_norm": 58.5,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.9762957096099854,
-      "logits/rejected": -1.9823601245880127,
-      "logps/chosen": -30.78021240234375,
-      "logps/rejected": -33.74589538574219,
-      "loss": 0.0957,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.38966792821884155,
-      "rewards/margins": 0.45797547698020935,
-      "rewards/rejected": -0.06830750405788422,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "grad_norm": 28.375,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.9236361980438232,
-      "logits/rejected": -1.9213138818740845,
-      "logps/chosen": -30.394033432006836,
-      "logps/rejected": -30.590625762939453,
-      "loss": 0.1267,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.40636783838272095,
-      "rewards/margins": 0.4031190872192383,
-      "rewards/rejected": 0.003248719498515129,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "grad_norm": 18.25,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -2.09725284576416,
-      "logits/rejected": -2.091542959213257,
-      "logps/chosen": -34.822288513183594,
-      "logps/rejected": -32.077293395996094,
-      "loss": 0.0885,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.4166763424873352,
-      "rewards/margins": 0.4352034032344818,
-      "rewards/rejected": -0.018527105450630188,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "grad_norm": 16.0,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.9885956048965454,
-      "logits/rejected": -1.9906467199325562,
-      "logps/chosen": -28.027481079101562,
-      "logps/rejected": -28.051410675048828,
-      "loss": 0.0847,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.40400925278663635,
-      "rewards/margins": 0.4302259087562561,
-      "rewards/rejected": -0.02621668018400669,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "grad_norm": 28.375,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.9982271194458008,
-      "logits/rejected": -1.9925235509872437,
-      "logps/chosen": -32.401649475097656,
-      "logps/rejected": -32.05929946899414,
-      "loss": 0.0757,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.359231173992157,
-      "rewards/margins": 0.44054365158081055,
-      "rewards/rejected": -0.08131246268749237,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.2469353675842285,
-      "eval_logits/rejected": -2.2420578002929688,
-      "eval_logps/chosen": -33.8865852355957,
-      "eval_logps/rejected": -37.384151458740234,
-      "eval_loss": 0.3542863130569458,
-      "eval_rewards/accuracies": 0.5485880374908447,
-      "eval_rewards/chosen": 0.1331672966480255,
-      "eval_rewards/margins": 0.013947191648185253,
-      "eval_rewards/rejected": 0.11922012269496918,
-      "eval_runtime": 145.2538,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "grad_norm": 30.375,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.9242042303085327,
-      "logits/rejected": -1.9285516738891602,
-      "logps/chosen": -33.746910095214844,
-      "logps/rejected": -34.46141815185547,
-      "loss": 0.1179,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.38167905807495117,
-      "rewards/margins": 0.4276120662689209,
-      "rewards/rejected": -0.04593303054571152,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "grad_norm": 78.5,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.9852806329727173,
-      "logits/rejected": -1.9835399389266968,
-      "logps/chosen": -30.321674346923828,
-      "logps/rejected": -32.68654251098633,
-      "loss": 0.1491,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3712538182735443,
-      "rewards/margins": 0.4583936631679535,
-      "rewards/rejected": -0.08713988214731216,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "grad_norm": 41.25,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -2.082426071166992,
-      "logits/rejected": -2.074831008911133,
-      "logps/chosen": -27.650592803955078,
-      "logps/rejected": -28.735992431640625,
-      "loss": 0.1085,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3711165487766266,
-      "rewards/margins": 0.4290649890899658,
-      "rewards/rejected": -0.05794848874211311,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 50.25,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.9804470539093018,
-      "logits/rejected": -1.9702799320220947,
-      "logps/chosen": -32.56533432006836,
-      "logps/rejected": -35.504661560058594,
-      "loss": 0.1078,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.4507782459259033,
-      "rewards/margins": 0.5349205732345581,
-      "rewards/rejected": -0.0841422826051712,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.17404492969636792,
-      "train_runtime": 10767.1792,
-      "train_samples_per_second": 1.144,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.8990784768934373,
+      "train_runtime": 3250.3805,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,