diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,18200 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 5.052942243062624,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.06865417957305908,
+      "logits/rejected": 0.14142072200775146,
+      "logps/chosen": -1.7158018350601196,
+      "logps/rejected": -1.8890635967254639,
+      "loss": 0.6062,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.7158018350601196,
+      "rewards/margins": 0.1732618361711502,
+      "rewards/rejected": -1.8890635967254639,
+      "sft_loss": 1.468388319015503,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 10.14750409435783,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": -0.0027637421153485775,
+      "logits/rejected": 0.11996668577194214,
+      "logps/chosen": -1.8019473552703857,
+      "logps/rejected": -1.8459587097167969,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8019473552703857,
+      "rewards/margins": 0.044011637568473816,
+      "rewards/rejected": -1.8459587097167969,
+      "sft_loss": 1.5082201957702637,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 11.385607782732158,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.041698895394802094,
+      "logits/rejected": 0.05798926204442978,
+      "logps/chosen": -1.6351089477539062,
+      "logps/rejected": -1.7640068531036377,
+      "loss": 0.6533,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6351089477539062,
+      "rewards/margins": 0.12889784574508667,
+      "rewards/rejected": -1.7640068531036377,
+      "sft_loss": 1.5005595684051514,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 5.148407815704086,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.04344125837087631,
+      "logits/rejected": 0.04649763181805611,
+      "logps/chosen": -1.7256911993026733,
+      "logps/rejected": -1.8059346675872803,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7256911993026733,
+      "rewards/margins": 0.08024374395608902,
+      "rewards/rejected": -1.8059346675872803,
+      "sft_loss": 1.500503659248352,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 16.345046525598722,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.05768662691116333,
+      "logits/rejected": 0.02871532365679741,
+      "logps/chosen": -1.8689944744110107,
+      "logps/rejected": -1.7781450748443604,
+      "loss": 0.7893,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -1.8689944744110107,
+      "rewards/margins": -0.09084945917129517,
+      "rewards/rejected": -1.7781450748443604,
+      "sft_loss": 1.5453780889511108,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 9.011620573094095,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.08053458482027054,
+      "logits/rejected": 0.014007637277245522,
+      "logps/chosen": -1.907224416732788,
+      "logps/rejected": -1.8314783573150635,
+      "loss": 0.7497,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -1.907224416732788,
+      "rewards/margins": -0.07574623078107834,
+      "rewards/rejected": -1.8314783573150635,
+      "sft_loss": 1.645878791809082,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 10.780963310335471,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.049357324838638306,
+      "logits/rejected": 0.11556194722652435,
+      "logps/chosen": -1.8474754095077515,
+      "logps/rejected": -1.9975019693374634,
+      "loss": 0.7166,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8474754095077515,
+      "rewards/margins": 0.1500265896320343,
+      "rewards/rejected": -1.9975019693374634,
+      "sft_loss": 1.5622758865356445,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 10.010044826292075,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.04069102555513382,
+      "logits/rejected": 0.21982774138450623,
+      "logps/chosen": -1.8808033466339111,
+      "logps/rejected": -1.7428737878799438,
+      "loss": 0.7596,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.8808033466339111,
+      "rewards/margins": -0.1379295140504837,
+      "rewards/rejected": -1.7428737878799438,
+      "sft_loss": 1.5191489458084106,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 15.791004960924981,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.015349939465522766,
+      "logits/rejected": 0.2134595811367035,
+      "logps/chosen": -1.835538625717163,
+      "logps/rejected": -1.8691489696502686,
+      "loss": 0.7282,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.835538625717163,
+      "rewards/margins": 0.03361044079065323,
+      "rewards/rejected": -1.8691489696502686,
+      "sft_loss": 1.535658359527588,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 12.493095326259008,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.06104832887649536,
+      "logits/rejected": 0.09160584956407547,
+      "logps/chosen": -1.8953063488006592,
+      "logps/rejected": -1.7769196033477783,
+      "loss": 0.7798,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.8953063488006592,
+      "rewards/margins": -0.11838690936565399,
+      "rewards/rejected": -1.7769196033477783,
+      "sft_loss": 1.5821568965911865,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 8.057579062835789,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.10174760967493057,
+      "logits/rejected": 0.12272598594427109,
+      "logps/chosen": -1.8287811279296875,
+      "logps/rejected": -1.8614591360092163,
+      "loss": 0.752,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8287811279296875,
+      "rewards/margins": 0.032678090035915375,
+      "rewards/rejected": -1.8614591360092163,
+      "sft_loss": 1.5816882848739624,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 7.542081099548166,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.08671236038208008,
+      "logits/rejected": 0.10760108381509781,
+      "logps/chosen": -1.7833054065704346,
+      "logps/rejected": -1.8888471126556396,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.7833054065704346,
+      "rewards/margins": 0.10554170608520508,
+      "rewards/rejected": -1.8888471126556396,
+      "sft_loss": 1.5421648025512695,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 6.531349763687479,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.04039953276515007,
+      "logits/rejected": 0.10648614168167114,
+      "logps/chosen": -1.6316719055175781,
+      "logps/rejected": -1.760345458984375,
+      "loss": 0.6193,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6316719055175781,
+      "rewards/margins": 0.1286735087633133,
+      "rewards/rejected": -1.760345458984375,
+      "sft_loss": 1.4724668264389038,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 12.333828845131993,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.07198110967874527,
+      "logits/rejected": 0.08535166829824448,
+      "logps/chosen": -1.7603833675384521,
+      "logps/rejected": -1.805906891822815,
+      "loss": 0.7312,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -1.7603833675384521,
+      "rewards/margins": 0.04552330821752548,
+      "rewards/rejected": -1.805906891822815,
+      "sft_loss": 1.6264307498931885,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 13.822532885111928,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.055731602013111115,
+      "logits/rejected": 0.1283172070980072,
+      "logps/chosen": -1.7673460245132446,
+      "logps/rejected": -2.0253078937530518,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7673460245132446,
+      "rewards/margins": 0.2579617202281952,
+      "rewards/rejected": -2.0253078937530518,
+      "sft_loss": 1.561607003211975,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 8.906433417273297,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": 0.009920082986354828,
+      "logits/rejected": 0.11612298339605331,
+      "logps/chosen": -1.6991240978240967,
+      "logps/rejected": -1.7324743270874023,
+      "loss": 0.682,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6991240978240967,
+      "rewards/margins": 0.033349912613630295,
+      "rewards/rejected": -1.7324743270874023,
+      "sft_loss": 1.51686692237854,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 5.494631905202502,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.14454609155654907,
+      "logits/rejected": 0.1083751767873764,
+      "logps/chosen": -1.7641146183013916,
+      "logps/rejected": -1.933570146560669,
+      "loss": 0.65,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7641146183013916,
+      "rewards/margins": 0.16945549845695496,
+      "rewards/rejected": -1.933570146560669,
+      "sft_loss": 1.4863570928573608,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 15.69779541487828,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.049393534660339355,
+      "logits/rejected": 0.011763748712837696,
+      "logps/chosen": -1.7164913415908813,
+      "logps/rejected": -1.7509660720825195,
+      "loss": 0.7072,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.7164913415908813,
+      "rewards/margins": 0.03447474539279938,
+      "rewards/rejected": -1.7509660720825195,
+      "sft_loss": 1.4483808279037476,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 6.508984995318275,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.07589919865131378,
+      "logits/rejected": 0.07544967532157898,
+      "logps/chosen": -1.7599709033966064,
+      "logps/rejected": -1.8785327672958374,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7599709033966064,
+      "rewards/margins": 0.11856192350387573,
+      "rewards/rejected": -1.8785327672958374,
+      "sft_loss": 1.5096004009246826,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 4.6447378329194215,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.03355532884597778,
+      "logits/rejected": 0.03286678344011307,
+      "logps/chosen": -1.6630899906158447,
+      "logps/rejected": -1.7689199447631836,
+      "loss": 0.642,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.6630899906158447,
+      "rewards/margins": 0.10582991689443588,
+      "rewards/rejected": -1.7689199447631836,
+      "sft_loss": 1.4793955087661743,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 10.397721008956594,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.04035579040646553,
+      "logits/rejected": 0.067632295191288,
+      "logps/chosen": -1.6082446575164795,
+      "logps/rejected": -1.7760941982269287,
+      "loss": 0.6146,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6082446575164795,
+      "rewards/margins": 0.16784964501857758,
+      "rewards/rejected": -1.7760941982269287,
+      "sft_loss": 1.4214261770248413,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 6.249475901409631,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": 0.0038494463078677654,
+      "logits/rejected": 0.1005120500922203,
+      "logps/chosen": -1.6075687408447266,
+      "logps/rejected": -1.6648200750350952,
+      "loss": 0.6582,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6075687408447266,
+      "rewards/margins": 0.0572517029941082,
+      "rewards/rejected": -1.6648200750350952,
+      "sft_loss": 1.4360030889511108,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 10.847905194112093,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.034572064876556396,
+      "logits/rejected": 0.2488282471895218,
+      "logps/chosen": -1.5851459503173828,
+      "logps/rejected": -1.8454182147979736,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5851459503173828,
+      "rewards/margins": 0.26027244329452515,
+      "rewards/rejected": -1.8454182147979736,
+      "sft_loss": 1.52280855178833,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 6.366957134835353,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.0667978897690773,
+      "logits/rejected": 0.11292694509029388,
+      "logps/chosen": -1.6360286474227905,
+      "logps/rejected": -1.7481434345245361,
+      "loss": 0.6288,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6360286474227905,
+      "rewards/margins": 0.11211474239826202,
+      "rewards/rejected": -1.7481434345245361,
+      "sft_loss": 1.5068069696426392,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 4.843145580690555,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.08813653141260147,
+      "logits/rejected": 0.044660698622465134,
+      "logps/chosen": -1.5761592388153076,
+      "logps/rejected": -1.541815996170044,
+      "loss": 0.672,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.5761592388153076,
+      "rewards/margins": -0.034343421459198,
+      "rewards/rejected": -1.541815996170044,
+      "sft_loss": 1.4881436824798584,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 8.927002657528016,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.009452106431126595,
+      "logits/rejected": 0.1407095491886139,
+      "logps/chosen": -1.6128228902816772,
+      "logps/rejected": -1.730139970779419,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.6128228902816772,
+      "rewards/margins": 0.11731685698032379,
+      "rewards/rejected": -1.730139970779419,
+      "sft_loss": 1.5444576740264893,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 16.0318675129108,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.07095210254192352,
+      "logits/rejected": 0.04487922415137291,
+      "logps/chosen": -1.6604677438735962,
+      "logps/rejected": -1.6921546459197998,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.6604677438735962,
+      "rewards/margins": 0.0316871777176857,
+      "rewards/rejected": -1.6921546459197998,
+      "sft_loss": 1.4799906015396118,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 7.8260263801664,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.05099118873476982,
+      "logits/rejected": 0.11860334873199463,
+      "logps/chosen": -1.6225074529647827,
+      "logps/rejected": -1.741729497909546,
+      "loss": 0.6204,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.6225074529647827,
+      "rewards/margins": 0.11922208964824677,
+      "rewards/rejected": -1.741729497909546,
+      "sft_loss": 1.5272603034973145,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 10.071220149350514,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": -0.02638971246778965,
+      "logits/rejected": 0.127262145280838,
+      "logps/chosen": -1.528851866722107,
+      "logps/rejected": -1.635650873184204,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.528851866722107,
+      "rewards/margins": 0.10679880529642105,
+      "rewards/rejected": -1.635650873184204,
+      "sft_loss": 1.4772685766220093,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 12.863350569849993,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.08558139950037003,
+      "logits/rejected": 0.07510876655578613,
+      "logps/chosen": -1.4721580743789673,
+      "logps/rejected": -1.472564935684204,
+      "loss": 0.64,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4721580743789673,
+      "rewards/margins": 0.000406736129662022,
+      "rewards/rejected": -1.472564935684204,
+      "sft_loss": 1.3356287479400635,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 8.630736310404208,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.07413525134325027,
+      "logits/rejected": -0.02334105595946312,
+      "logps/chosen": -1.4584012031555176,
+      "logps/rejected": -1.562864065170288,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4584012031555176,
+      "rewards/margins": 0.10446293652057648,
+      "rewards/rejected": -1.562864065170288,
+      "sft_loss": 1.4039736986160278,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 8.976834922671944,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.18383175134658813,
+      "logits/rejected": -0.04810567945241928,
+      "logps/chosen": -1.522956371307373,
+      "logps/rejected": -1.5014146566390991,
+      "loss": 0.661,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.522956371307373,
+      "rewards/margins": -0.021541709080338478,
+      "rewards/rejected": -1.5014146566390991,
+      "sft_loss": 1.433183193206787,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 7.445472741165519,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.10111711174249649,
+      "logits/rejected": 0.06512321531772614,
+      "logps/chosen": -1.3696064949035645,
+      "logps/rejected": -1.4571757316589355,
+      "loss": 0.595,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3696064949035645,
+      "rewards/margins": 0.08756937086582184,
+      "rewards/rejected": -1.4571757316589355,
+      "sft_loss": 1.324439287185669,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 14.656981338791415,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.11693856865167618,
+      "logits/rejected": -0.06399437040090561,
+      "logps/chosen": -1.4670077562332153,
+      "logps/rejected": -1.536694049835205,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.4670077562332153,
+      "rewards/margins": 0.06968621164560318,
+      "rewards/rejected": -1.536694049835205,
+      "sft_loss": 1.440754771232605,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 6.584009251136716,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": -0.006804236676543951,
+      "logits/rejected": -0.009001683443784714,
+      "logps/chosen": -1.3459522724151611,
+      "logps/rejected": -1.4413591623306274,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3459522724151611,
+      "rewards/margins": 0.09540703147649765,
+      "rewards/rejected": -1.4413591623306274,
+      "sft_loss": 1.368872046470642,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 6.038444404374204,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": -0.06775864958763123,
+      "logits/rejected": -0.06924072653055191,
+      "logps/chosen": -1.3591883182525635,
+      "logps/rejected": -1.560471773147583,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3591883182525635,
+      "rewards/margins": 0.20128345489501953,
+      "rewards/rejected": -1.560471773147583,
+      "sft_loss": 1.378900408744812,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 6.851019247142483,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.20338714122772217,
+      "logits/rejected": -0.1196417361497879,
+      "logps/chosen": -1.3481305837631226,
+      "logps/rejected": -1.3971184492111206,
+      "loss": 0.601,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.3481305837631226,
+      "rewards/margins": 0.04898771643638611,
+      "rewards/rejected": -1.3971184492111206,
+      "sft_loss": 1.3574292659759521,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 6.199864509350299,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": -0.11407258361577988,
+      "logits/rejected": -0.0008642614120617509,
+      "logps/chosen": -1.2794859409332275,
+      "logps/rejected": -1.4109026193618774,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2794859409332275,
+      "rewards/margins": 0.13141660392284393,
+      "rewards/rejected": -1.4109026193618774,
+      "sft_loss": 1.2981491088867188,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 4.548236551688456,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": -0.05843614414334297,
+      "logits/rejected": 0.08690853416919708,
+      "logps/chosen": -1.2500104904174805,
+      "logps/rejected": -1.4155646562576294,
+      "loss": 0.528,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2500104904174805,
+      "rewards/margins": 0.16555407643318176,
+      "rewards/rejected": -1.4155646562576294,
+      "sft_loss": 1.2920830249786377,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 15.793009061938657,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.15543696284294128,
+      "logits/rejected": -0.02773769572377205,
+      "logps/chosen": -1.3737095594406128,
+      "logps/rejected": -1.413891077041626,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3737095594406128,
+      "rewards/margins": 0.04018155485391617,
+      "rewards/rejected": -1.413891077041626,
+      "sft_loss": 1.3922524452209473,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 10.870168642235853,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.0997370257973671,
+      "logits/rejected": 0.03656370937824249,
+      "logps/chosen": -1.2865569591522217,
+      "logps/rejected": -1.3560245037078857,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2865569591522217,
+      "rewards/margins": 0.06946732848882675,
+      "rewards/rejected": -1.3560245037078857,
+      "sft_loss": 1.2838119268417358,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 7.032884322797753,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.2042866200208664,
+      "logits/rejected": -0.028805147856473923,
+      "logps/chosen": -1.360367774963379,
+      "logps/rejected": -1.473173975944519,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.360367774963379,
+      "rewards/margins": 0.11280622333288193,
+      "rewards/rejected": -1.473173975944519,
+      "sft_loss": 1.3383769989013672,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 5.142657637211093,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.22696277499198914,
+      "logits/rejected": 0.007920200005173683,
+      "logps/chosen": -1.3814806938171387,
+      "logps/rejected": -1.436943769454956,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3814806938171387,
+      "rewards/margins": 0.05546308681368828,
+      "rewards/rejected": -1.436943769454956,
+      "sft_loss": 1.3624300956726074,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 13.237933869494071,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": -0.008888229727745056,
+      "logits/rejected": 0.08174064010381699,
+      "logps/chosen": -1.306950569152832,
+      "logps/rejected": -1.4514024257659912,
+      "loss": 0.561,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.306950569152832,
+      "rewards/margins": 0.14445172250270844,
+      "rewards/rejected": -1.4514024257659912,
+      "sft_loss": 1.3328325748443604,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 4.585259904611552,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.15518362820148468,
+      "logits/rejected": 0.0046308608725667,
+      "logps/chosen": -1.3100550174713135,
+      "logps/rejected": -1.4391072988510132,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3100550174713135,
+      "rewards/margins": 0.12905225157737732,
+      "rewards/rejected": -1.4391072988510132,
+      "sft_loss": 1.3125994205474854,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 4.796634954673317,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": -0.06799010187387466,
+      "logits/rejected": 0.005717620253562927,
+      "logps/chosen": -1.3139328956604004,
+      "logps/rejected": -1.4714056253433228,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3139328956604004,
+      "rewards/margins": 0.15747258067131042,
+      "rewards/rejected": -1.4714056253433228,
+      "sft_loss": 1.2739067077636719,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 6.068497866740446,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": -0.037981465458869934,
+      "logits/rejected": 0.08764432370662689,
+      "logps/chosen": -1.2823946475982666,
+      "logps/rejected": -1.4488177299499512,
+      "loss": 0.5325,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2823946475982666,
+      "rewards/margins": 0.16642294824123383,
+      "rewards/rejected": -1.4488177299499512,
+      "sft_loss": 1.2844218015670776,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 4.365912398384617,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": -0.09100799262523651,
+      "logits/rejected": 0.025594225153326988,
+      "logps/chosen": -1.297890543937683,
+      "logps/rejected": -1.481879711151123,
+      "loss": 0.5342,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.297890543937683,
+      "rewards/margins": 0.18398909270763397,
+      "rewards/rejected": -1.481879711151123,
+      "sft_loss": 1.3451718091964722,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 6.346243072159388,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": -0.007030495908111334,
+      "logits/rejected": 0.10519347339868546,
+      "logps/chosen": -1.4173331260681152,
+      "logps/rejected": -1.44147527217865,
+      "loss": 0.6128,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4173331260681152,
+      "rewards/margins": 0.024142302572727203,
+      "rewards/rejected": -1.44147527217865,
+      "sft_loss": 1.4315967559814453,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 8.178991051627037,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": -0.11276876926422119,
+      "logits/rejected": 0.04048747569322586,
+      "logps/chosen": -1.2980380058288574,
+      "logps/rejected": -1.3559579849243164,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.2980380058288574,
+      "rewards/margins": 0.05791999027132988,
+      "rewards/rejected": -1.3559579849243164,
+      "sft_loss": 1.3147931098937988,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 5.943792587616856,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -0.051166605204343796,
+      "logits/rejected": 0.08564223349094391,
+      "logps/chosen": -1.2638366222381592,
+      "logps/rejected": -1.3699913024902344,
+      "loss": 0.5457,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2638366222381592,
+      "rewards/margins": 0.106154665350914,
+      "rewards/rejected": -1.3699913024902344,
+      "sft_loss": 1.2442152500152588,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 5.011172706823575,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.26364797353744507,
+      "logits/rejected": -0.1617596596479416,
+      "logps/chosen": -1.3498268127441406,
+      "logps/rejected": -1.5086586475372314,
+      "loss": 0.5425,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3498268127441406,
+      "rewards/margins": 0.15883179008960724,
+      "rewards/rejected": -1.5086586475372314,
+      "sft_loss": 1.3699219226837158,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 5.581957559969404,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": -0.11523625999689102,
+      "logits/rejected": -0.029855186119675636,
+      "logps/chosen": -1.3342697620391846,
+      "logps/rejected": -1.5049277544021606,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3342697620391846,
+      "rewards/margins": 0.17065785825252533,
+      "rewards/rejected": -1.5049277544021606,
+      "sft_loss": 1.3940002918243408,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 3.8784079567677807,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": -0.13867095112800598,
+      "logits/rejected": -0.01858574151992798,
+      "logps/chosen": -1.315878987312317,
+      "logps/rejected": -1.4139220714569092,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.315878987312317,
+      "rewards/margins": 0.09804315865039825,
+      "rewards/rejected": -1.4139220714569092,
+      "sft_loss": 1.334929347038269,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 4.914311906008871,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": -0.06731410324573517,
+      "logits/rejected": 0.025944000110030174,
+      "logps/chosen": -1.2634482383728027,
+      "logps/rejected": -1.4206976890563965,
+      "loss": 0.5289,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2634482383728027,
+      "rewards/margins": 0.15724948048591614,
+      "rewards/rejected": -1.4206976890563965,
+      "sft_loss": 1.2450734376907349,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 5.175769918745935,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": -0.13446274399757385,
+      "logits/rejected": 0.018282266333699226,
+      "logps/chosen": -1.3150831460952759,
+      "logps/rejected": -1.414188027381897,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3150831460952759,
+      "rewards/margins": 0.09910491853952408,
+      "rewards/rejected": -1.414188027381897,
+      "sft_loss": 1.3091659545898438,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 5.877826446213898,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": -0.11841098964214325,
+      "logits/rejected": 0.011662433855235577,
+      "logps/chosen": -1.3436254262924194,
+      "logps/rejected": -1.4265941381454468,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3436254262924194,
+      "rewards/margins": 0.08296870440244675,
+      "rewards/rejected": -1.4265941381454468,
+      "sft_loss": 1.3912488222122192,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 5.694699355384885,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": -0.14523978531360626,
+      "logits/rejected": 0.14297033846378326,
+      "logps/chosen": -1.3640129566192627,
+      "logps/rejected": -1.48271644115448,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3640129566192627,
+      "rewards/margins": 0.1187034398317337,
+      "rewards/rejected": -1.48271644115448,
+      "sft_loss": 1.3594133853912354,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 5.834241126679447,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": -0.09991417825222015,
+      "logits/rejected": -0.04301440715789795,
+      "logps/chosen": -1.2645776271820068,
+      "logps/rejected": -1.4023511409759521,
+      "loss": 0.5371,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2645776271820068,
+      "rewards/margins": 0.13777343928813934,
+      "rewards/rejected": -1.4023511409759521,
+      "sft_loss": 1.2729285955429077,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 5.597124434517011,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": -0.09893033653497696,
+      "logits/rejected": 0.06294500082731247,
+      "logps/chosen": -1.301417350769043,
+      "logps/rejected": -1.3780789375305176,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.301417350769043,
+      "rewards/margins": 0.07666166126728058,
+      "rewards/rejected": -1.3780789375305176,
+      "sft_loss": 1.3655966520309448,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 4.26674515151328,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": -0.06475608050823212,
+      "logits/rejected": 0.005105156451463699,
+      "logps/chosen": -1.406415581703186,
+      "logps/rejected": -1.4127743244171143,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.406415581703186,
+      "rewards/margins": 0.006358677055686712,
+      "rewards/rejected": -1.4127743244171143,
+      "sft_loss": 1.410060167312622,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 6.211235277860888,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.2585568428039551,
+      "logits/rejected": -0.1752055436372757,
+      "logps/chosen": -1.3687752485275269,
+      "logps/rejected": -1.4534339904785156,
+      "loss": 0.5973,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3687752485275269,
+      "rewards/margins": 0.0846589207649231,
+      "rewards/rejected": -1.4534339904785156,
+      "sft_loss": 1.3693883419036865,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 6.564026888050156,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": -0.04630298539996147,
+      "logits/rejected": 0.1092165932059288,
+      "logps/chosen": -1.3590734004974365,
+      "logps/rejected": -1.5081905126571655,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3590734004974365,
+      "rewards/margins": 0.149117112159729,
+      "rewards/rejected": -1.5081905126571655,
+      "sft_loss": 1.3778035640716553,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 4.348469697914348,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": -0.09974730014801025,
+      "logits/rejected": 0.02833659015595913,
+      "logps/chosen": -1.3171430826187134,
+      "logps/rejected": -1.3695036172866821,
+      "loss": 0.5721,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3171430826187134,
+      "rewards/margins": 0.052360523492097855,
+      "rewards/rejected": -1.3695036172866821,
+      "sft_loss": 1.339951992034912,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 5.105999096976248,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.16527916491031647,
+      "logits/rejected": -0.05536722391843796,
+      "logps/chosen": -1.3153146505355835,
+      "logps/rejected": -1.5632933378219604,
+      "loss": 0.5331,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3153146505355835,
+      "rewards/margins": 0.24797877669334412,
+      "rewards/rejected": -1.5632933378219604,
+      "sft_loss": 1.4050966501235962,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 8.82419196921446,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": -0.07711423933506012,
+      "logits/rejected": 0.06198772042989731,
+      "logps/chosen": -1.3323601484298706,
+      "logps/rejected": -1.50588059425354,
+      "loss": 0.55,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3323601484298706,
+      "rewards/margins": 0.17352035641670227,
+      "rewards/rejected": -1.50588059425354,
+      "sft_loss": 1.3518102169036865,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 8.09083744856097,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.0038772523403167725,
+      "logits/rejected": 0.10497460514307022,
+      "logps/chosen": -1.3427479267120361,
+      "logps/rejected": -1.3789106607437134,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3427479267120361,
+      "rewards/margins": 0.036162860691547394,
+      "rewards/rejected": -1.3789106607437134,
+      "sft_loss": 1.3472321033477783,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 7.465959336338128,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": -0.05871904641389847,
+      "logits/rejected": 0.08265434950590134,
+      "logps/chosen": -1.4018275737762451,
+      "logps/rejected": -1.488533854484558,
+      "loss": 0.5994,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4018275737762451,
+      "rewards/margins": 0.08670647442340851,
+      "rewards/rejected": -1.488533854484558,
+      "sft_loss": 1.3964767456054688,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 8.49429791242324,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.0036362111568450928,
+      "logits/rejected": 0.03215277940034866,
+      "logps/chosen": -1.3091847896575928,
+      "logps/rejected": -1.4534891843795776,
+      "loss": 0.5495,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3091847896575928,
+      "rewards/margins": 0.1443043351173401,
+      "rewards/rejected": -1.4534891843795776,
+      "sft_loss": 1.3581584692001343,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 5.6762514137157165,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": -0.04075011983513832,
+      "logits/rejected": 0.05240970104932785,
+      "logps/chosen": -1.2877025604248047,
+      "logps/rejected": -1.4130481481552124,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2877025604248047,
+      "rewards/margins": 0.12534549832344055,
+      "rewards/rejected": -1.4130481481552124,
+      "sft_loss": 1.350273847579956,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 5.6317484450607225,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": -0.14434251189231873,
+      "logits/rejected": 0.06206267327070236,
+      "logps/chosen": -1.3909976482391357,
+      "logps/rejected": -1.4239745140075684,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3909976482391357,
+      "rewards/margins": 0.032976895570755005,
+      "rewards/rejected": -1.4239745140075684,
+      "sft_loss": 1.4065606594085693,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 5.730528263535874,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": -0.10777083784341812,
+      "logits/rejected": -0.0323818065226078,
+      "logps/chosen": -1.3194472789764404,
+      "logps/rejected": -1.4491716623306274,
+      "loss": 0.5493,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3194472789764404,
+      "rewards/margins": 0.12972450256347656,
+      "rewards/rejected": -1.4491716623306274,
+      "sft_loss": 1.3073797225952148,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 7.441457504970106,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": -0.03163684159517288,
+      "logits/rejected": 0.046002812683582306,
+      "logps/chosen": -1.2995356321334839,
+      "logps/rejected": -1.405275821685791,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2995356321334839,
+      "rewards/margins": 0.10574007034301758,
+      "rewards/rejected": -1.405275821685791,
+      "sft_loss": 1.2813327312469482,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 4.578902430418264,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": -0.049534980207681656,
+      "logits/rejected": 0.04207900911569595,
+      "logps/chosen": -1.2937860488891602,
+      "logps/rejected": -1.3419485092163086,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2937860488891602,
+      "rewards/margins": 0.048162512481212616,
+      "rewards/rejected": -1.3419485092163086,
+      "sft_loss": 1.2689062356948853,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 5.977001125118096,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": -0.10349956899881363,
+      "logits/rejected": 0.04922555759549141,
+      "logps/chosen": -1.2733392715454102,
+      "logps/rejected": -1.4207669496536255,
+      "loss": 0.546,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2733392715454102,
+      "rewards/margins": 0.14742770791053772,
+      "rewards/rejected": -1.4207669496536255,
+      "sft_loss": 1.3344981670379639,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 4.7894362626255,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": -0.05292842537164688,
+      "logits/rejected": 0.030161788687109947,
+      "logps/chosen": -1.2897326946258545,
+      "logps/rejected": -1.4556329250335693,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2897326946258545,
+      "rewards/margins": 0.16590023040771484,
+      "rewards/rejected": -1.4556329250335693,
+      "sft_loss": 1.305248498916626,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 4.0595535401027645,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": -0.0235292986035347,
+      "logits/rejected": 0.05328557640314102,
+      "logps/chosen": -1.3905055522918701,
+      "logps/rejected": -1.3857368230819702,
+      "loss": 0.6318,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3905055522918701,
+      "rewards/margins": -0.004768743179738522,
+      "rewards/rejected": -1.3857368230819702,
+      "sft_loss": 1.4026410579681396,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 7.727865438010638,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": 0.042860690504312515,
+      "logits/rejected": 0.2084215134382248,
+      "logps/chosen": -1.383760690689087,
+      "logps/rejected": -1.4487297534942627,
+      "loss": 0.601,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.383760690689087,
+      "rewards/margins": 0.06496912240982056,
+      "rewards/rejected": -1.4487297534942627,
+      "sft_loss": 1.392791748046875,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 5.87319636795186,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": -0.07385215908288956,
+      "logits/rejected": 0.08165531605482101,
+      "logps/chosen": -1.3311251401901245,
+      "logps/rejected": -1.3491557836532593,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3311251401901245,
+      "rewards/margins": 0.01803075149655342,
+      "rewards/rejected": -1.3491557836532593,
+      "sft_loss": 1.3450191020965576,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 4.475996032871945,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": 0.04886271804571152,
+      "logits/rejected": 0.1430073380470276,
+      "logps/chosen": -1.3099935054779053,
+      "logps/rejected": -1.4135615825653076,
+      "loss": 0.5602,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3099935054779053,
+      "rewards/margins": 0.1035681962966919,
+      "rewards/rejected": -1.4135615825653076,
+      "sft_loss": 1.3142729997634888,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": 0.252483993768692,
+      "eval_logits/rejected": 0.3397251069545746,
+      "eval_logps/chosen": -1.3432023525238037,
+      "eval_logps/rejected": -1.480107069015503,
+      "eval_loss": 0.5592606663703918,
+      "eval_rewards/accuracies": 0.5578634738922119,
+      "eval_rewards/chosen": -1.3432023525238037,
+      "eval_rewards/margins": 0.13690461218357086,
+      "eval_rewards/rejected": -1.480107069015503,
+      "eval_runtime": 43.9397,
+      "eval_samples_per_second": 30.61,
+      "eval_sft_loss": 1.370203971862793,
+      "eval_steps_per_second": 7.67,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 7.034234666380894,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": -0.027093514800071716,
+      "logits/rejected": 0.06692580878734589,
+      "logps/chosen": -1.3181028366088867,
+      "logps/rejected": -1.389564871788025,
+      "loss": 0.5726,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3181028366088867,
+      "rewards/margins": 0.07146209478378296,
+      "rewards/rejected": -1.389564871788025,
+      "sft_loss": 1.316383957862854,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 4.841587816363229,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": 0.003671199083328247,
+      "logits/rejected": 0.13374844193458557,
+      "logps/chosen": -1.2869586944580078,
+      "logps/rejected": -1.382009506225586,
+      "loss": 0.5626,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2869586944580078,
+      "rewards/margins": 0.09505093097686768,
+      "rewards/rejected": -1.382009506225586,
+      "sft_loss": 1.3342974185943604,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 4.276205679053103,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": -0.01809159852564335,
+      "logits/rejected": 0.020652102306485176,
+      "logps/chosen": -1.28365957736969,
+      "logps/rejected": -1.4585318565368652,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.28365957736969,
+      "rewards/margins": 0.17487232387065887,
+      "rewards/rejected": -1.4585318565368652,
+      "sft_loss": 1.3000603914260864,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 5.072655211143472,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": -0.03806052356958389,
+      "logits/rejected": 0.14919230341911316,
+      "logps/chosen": -1.273961067199707,
+      "logps/rejected": -1.3729407787322998,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.273961067199707,
+      "rewards/margins": 0.09897960722446442,
+      "rewards/rejected": -1.3729407787322998,
+      "sft_loss": 1.3288816213607788,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 4.995296057342031,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": -0.06995834410190582,
+      "logits/rejected": 0.1246950775384903,
+      "logps/chosen": -1.313232660293579,
+      "logps/rejected": -1.4796546697616577,
+      "loss": 0.5382,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.313232660293579,
+      "rewards/margins": 0.16642197966575623,
+      "rewards/rejected": -1.4796546697616577,
+      "sft_loss": 1.3914697170257568,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 5.318848179106577,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": -0.09253229945898056,
+      "logits/rejected": 0.10250599682331085,
+      "logps/chosen": -1.3457051515579224,
+      "logps/rejected": -1.4839556217193604,
+      "loss": 0.5495,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3457051515579224,
+      "rewards/margins": 0.1382504254579544,
+      "rewards/rejected": -1.4839556217193604,
+      "sft_loss": 1.3906058073043823,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 6.2690432389327375,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": -0.028032371774315834,
+      "logits/rejected": 0.0593259334564209,
+      "logps/chosen": -1.2195804119110107,
+      "logps/rejected": -1.3592599630355835,
+      "loss": 0.5264,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2195804119110107,
+      "rewards/margins": 0.13967978954315186,
+      "rewards/rejected": -1.3592599630355835,
+      "sft_loss": 1.2889639139175415,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 4.38473816919049,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": -0.03974657505750656,
+      "logits/rejected": 0.04667690023779869,
+      "logps/chosen": -1.2953468561172485,
+      "logps/rejected": -1.3932148218154907,
+      "loss": 0.5548,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2953468561172485,
+      "rewards/margins": 0.09786802530288696,
+      "rewards/rejected": -1.3932148218154907,
+      "sft_loss": 1.3165804147720337,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 5.461131482304206,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": -0.06804539263248444,
+      "logits/rejected": 0.042102426290512085,
+      "logps/chosen": -1.324836254119873,
+      "logps/rejected": -1.4605040550231934,
+      "loss": 0.57,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.324836254119873,
+      "rewards/margins": 0.13566775619983673,
+      "rewards/rejected": -1.4605040550231934,
+      "sft_loss": 1.3660962581634521,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 7.46942560698112,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": -0.02956671081483364,
+      "logits/rejected": 0.09420625865459442,
+      "logps/chosen": -1.3139501810073853,
+      "logps/rejected": -1.4573780298233032,
+      "loss": 0.5296,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3139501810073853,
+      "rewards/margins": 0.14342793822288513,
+      "rewards/rejected": -1.4573780298233032,
+      "sft_loss": 1.301524043083191,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 5.766091566763157,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": -0.008473304100334644,
+      "logits/rejected": 0.08258920907974243,
+      "logps/chosen": -1.2512413263320923,
+      "logps/rejected": -1.4463129043579102,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2512413263320923,
+      "rewards/margins": 0.1950714886188507,
+      "rewards/rejected": -1.4463129043579102,
+      "sft_loss": 1.2772114276885986,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 5.822042326452618,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": -0.13016703724861145,
+      "logits/rejected": -0.006366625428199768,
+      "logps/chosen": -1.378727912902832,
+      "logps/rejected": -1.4377683401107788,
+      "loss": 0.594,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.378727912902832,
+      "rewards/margins": 0.059040505439043045,
+      "rewards/rejected": -1.4377683401107788,
+      "sft_loss": 1.4157121181488037,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 6.125179452056327,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": 0.12094781547784805,
+      "logits/rejected": 0.14069999754428864,
+      "logps/chosen": -1.286008596420288,
+      "logps/rejected": -1.4609568119049072,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.286008596420288,
+      "rewards/margins": 0.1749483048915863,
+      "rewards/rejected": -1.4609568119049072,
+      "sft_loss": 1.2957916259765625,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 6.032620914644976,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": 0.14457985758781433,
+      "logits/rejected": 0.10018442571163177,
+      "logps/chosen": -1.2448914051055908,
+      "logps/rejected": -1.4428813457489014,
+      "loss": 0.5237,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2448914051055908,
+      "rewards/margins": 0.19798986613750458,
+      "rewards/rejected": -1.4428813457489014,
+      "sft_loss": 1.2870714664459229,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 5.5865279276737665,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": -0.07837694883346558,
+      "logits/rejected": 0.06283046305179596,
+      "logps/chosen": -1.3094193935394287,
+      "logps/rejected": -1.5276954174041748,
+      "loss": 0.5274,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3094193935394287,
+      "rewards/margins": 0.2182762175798416,
+      "rewards/rejected": -1.5276954174041748,
+      "sft_loss": 1.3427727222442627,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 5.435895867877187,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": -0.055062223225831985,
+      "logits/rejected": 0.14435690641403198,
+      "logps/chosen": -1.2810598611831665,
+      "logps/rejected": -1.3541405200958252,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2810598611831665,
+      "rewards/margins": 0.07308082282543182,
+      "rewards/rejected": -1.3541405200958252,
+      "sft_loss": 1.3179230690002441,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 7.476411782783797,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": -0.0025172666646540165,
+      "logits/rejected": 0.03772548958659172,
+      "logps/chosen": -1.3743191957473755,
+      "logps/rejected": -1.4703030586242676,
+      "loss": 0.5768,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3743191957473755,
+      "rewards/margins": 0.09598368406295776,
+      "rewards/rejected": -1.4703030586242676,
+      "sft_loss": 1.3797461986541748,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 5.044411300219118,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": -0.021833175793290138,
+      "logits/rejected": 0.050391364842653275,
+      "logps/chosen": -1.3183096647262573,
+      "logps/rejected": -1.3980544805526733,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3183096647262573,
+      "rewards/margins": 0.07974480092525482,
+      "rewards/rejected": -1.3980544805526733,
+      "sft_loss": 1.319129228591919,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 6.6379109376317516,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": -0.05253653600811958,
+      "logits/rejected": -0.027758020907640457,
+      "logps/chosen": -1.3289823532104492,
+      "logps/rejected": -1.44633948802948,
+      "loss": 0.5614,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3289823532104492,
+      "rewards/margins": 0.11735711246728897,
+      "rewards/rejected": -1.44633948802948,
+      "sft_loss": 1.3961108922958374,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 5.349956993841474,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": -0.05144606903195381,
+      "logits/rejected": 0.048278652131557465,
+      "logps/chosen": -1.235568642616272,
+      "logps/rejected": -1.3974435329437256,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.235568642616272,
+      "rewards/margins": 0.16187506914138794,
+      "rewards/rejected": -1.3974435329437256,
+      "sft_loss": 1.2869036197662354,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 8.559922082272234,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": -0.05748797580599785,
+      "logits/rejected": 0.08330624550580978,
+      "logps/chosen": -1.3548730611801147,
+      "logps/rejected": -1.4059706926345825,
+      "loss": 0.5894,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3548730611801147,
+      "rewards/margins": 0.05109750106930733,
+      "rewards/rejected": -1.4059706926345825,
+      "sft_loss": 1.38583242893219,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 5.775228165472396,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": 0.09095264971256256,
+      "logits/rejected": 0.15189726650714874,
+      "logps/chosen": -1.3080486059188843,
+      "logps/rejected": -1.485636591911316,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3080486059188843,
+      "rewards/margins": 0.1775878518819809,
+      "rewards/rejected": -1.485636591911316,
+      "sft_loss": 1.2965028285980225,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 4.144221629537718,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": 0.05798070505261421,
+      "logits/rejected": 0.15644438564777374,
+      "logps/chosen": -1.249235987663269,
+      "logps/rejected": -1.4133098125457764,
+      "loss": 0.5357,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.249235987663269,
+      "rewards/margins": 0.16407367587089539,
+      "rewards/rejected": -1.4133098125457764,
+      "sft_loss": 1.2908960580825806,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 4.279287128656563,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": -0.0792783722281456,
+      "logits/rejected": 0.0598064549267292,
+      "logps/chosen": -1.2978101968765259,
+      "logps/rejected": -1.4097940921783447,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2978101968765259,
+      "rewards/margins": 0.1119840145111084,
+      "rewards/rejected": -1.4097940921783447,
+      "sft_loss": 1.4094016551971436,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 10.53445430162785,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": 0.1093759760260582,
+      "logits/rejected": 0.18354789912700653,
+      "logps/chosen": -1.275532841682434,
+      "logps/rejected": -1.4687607288360596,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.275532841682434,
+      "rewards/margins": 0.193228080868721,
+      "rewards/rejected": -1.4687607288360596,
+      "sft_loss": 1.373024344444275,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 4.390986869790019,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": 0.08234995603561401,
+      "logits/rejected": 0.1688038408756256,
+      "logps/chosen": -1.2554372549057007,
+      "logps/rejected": -1.3672778606414795,
+      "loss": 0.5493,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2554372549057007,
+      "rewards/margins": 0.11184068024158478,
+      "rewards/rejected": -1.3672778606414795,
+      "sft_loss": 1.230678915977478,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 4.469595540150405,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": -0.08351071178913116,
+      "logits/rejected": 0.18095479905605316,
+      "logps/chosen": -1.2669923305511475,
+      "logps/rejected": -1.3454608917236328,
+      "loss": 0.553,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2669923305511475,
+      "rewards/margins": 0.07846838980913162,
+      "rewards/rejected": -1.3454608917236328,
+      "sft_loss": 1.253920555114746,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 4.434656541244071,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": 0.04407946765422821,
+      "logits/rejected": 0.1207633838057518,
+      "logps/chosen": -1.396340250968933,
+      "logps/rejected": -1.4665273427963257,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.396340250968933,
+      "rewards/margins": 0.07018700987100601,
+      "rewards/rejected": -1.4665273427963257,
+      "sft_loss": 1.4373433589935303,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 4.136588625276095,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": -0.07348589599132538,
+      "logits/rejected": 0.1302313506603241,
+      "logps/chosen": -1.299394965171814,
+      "logps/rejected": -1.4257206916809082,
+      "loss": 0.5352,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.299394965171814,
+      "rewards/margins": 0.12632553279399872,
+      "rewards/rejected": -1.4257206916809082,
+      "sft_loss": 1.324333906173706,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 4.352118373128978,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": 0.07032302767038345,
+      "logits/rejected": 0.14239667356014252,
+      "logps/chosen": -1.2955199480056763,
+      "logps/rejected": -1.4420729875564575,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2955199480056763,
+      "rewards/margins": 0.1465531587600708,
+      "rewards/rejected": -1.4420729875564575,
+      "sft_loss": 1.2931840419769287,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 6.622239099763008,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": -0.059581220149993896,
+      "logits/rejected": 0.06685901433229446,
+      "logps/chosen": -1.3705341815948486,
+      "logps/rejected": -1.4440644979476929,
+      "loss": 0.595,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3705341815948486,
+      "rewards/margins": 0.0735301673412323,
+      "rewards/rejected": -1.4440644979476929,
+      "sft_loss": 1.3887046575546265,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 5.616977527800097,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": 0.03694029897451401,
+      "logits/rejected": 0.05506284162402153,
+      "logps/chosen": -1.2274644374847412,
+      "logps/rejected": -1.3752453327178955,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2274644374847412,
+      "rewards/margins": 0.14778101444244385,
+      "rewards/rejected": -1.3752453327178955,
+      "sft_loss": 1.3644822835922241,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 3.896688294517457,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": 0.033395491540431976,
+      "logits/rejected": 0.09102457761764526,
+      "logps/chosen": -1.3448399305343628,
+      "logps/rejected": -1.4782801866531372,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3448399305343628,
+      "rewards/margins": 0.13344022631645203,
+      "rewards/rejected": -1.4782801866531372,
+      "sft_loss": 1.3826723098754883,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 6.468949859109518,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": -0.03206388279795647,
+      "logits/rejected": 0.1903473287820816,
+      "logps/chosen": -1.3156042098999023,
+      "logps/rejected": -1.4189167022705078,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3156042098999023,
+      "rewards/margins": 0.10331261157989502,
+      "rewards/rejected": -1.4189167022705078,
+      "sft_loss": 1.4111707210540771,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 4.715156501261207,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": -0.016568869352340698,
+      "logits/rejected": 0.04429225996136665,
+      "logps/chosen": -1.2501987218856812,
+      "logps/rejected": -1.3790241479873657,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2501987218856812,
+      "rewards/margins": 0.128825381398201,
+      "rewards/rejected": -1.3790241479873657,
+      "sft_loss": 1.2782385349273682,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 5.367916524502688,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": -0.01917845942080021,
+      "logits/rejected": 0.1418890506029129,
+      "logps/chosen": -1.2578179836273193,
+      "logps/rejected": -1.440937876701355,
+      "loss": 0.5176,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2578179836273193,
+      "rewards/margins": 0.18311990797519684,
+      "rewards/rejected": -1.440937876701355,
+      "sft_loss": 1.2896482944488525,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 5.914173986510269,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": -0.07606562972068787,
+      "logits/rejected": 0.027116943150758743,
+      "logps/chosen": -1.3965537548065186,
+      "logps/rejected": -1.4706159830093384,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3965537548065186,
+      "rewards/margins": 0.07406215369701385,
+      "rewards/rejected": -1.4706159830093384,
+      "sft_loss": 1.4381605386734009,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 6.775658620216753,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": -0.04157830402255058,
+      "logits/rejected": 0.08735966682434082,
+      "logps/chosen": -1.3702585697174072,
+      "logps/rejected": -1.4163644313812256,
+      "loss": 0.5928,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3702585697174072,
+      "rewards/margins": 0.0461057610809803,
+      "rewards/rejected": -1.4163644313812256,
+      "sft_loss": 1.413253664970398,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 5.304206518820304,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": 0.021392468363046646,
+      "logits/rejected": 0.14886918663978577,
+      "logps/chosen": -1.3005200624465942,
+      "logps/rejected": -1.3912214040756226,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3005200624465942,
+      "rewards/margins": 0.09070131927728653,
+      "rewards/rejected": -1.3912214040756226,
+      "sft_loss": 1.318687915802002,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 4.73947769164422,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": -0.04575077071785927,
+      "logits/rejected": 0.10018666833639145,
+      "logps/chosen": -1.2980984449386597,
+      "logps/rejected": -1.3406703472137451,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.2980984449386597,
+      "rewards/margins": 0.0425720252096653,
+      "rewards/rejected": -1.3406703472137451,
+      "sft_loss": 1.3395038843154907,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 5.043264487838186,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": -0.08167050778865814,
+      "logits/rejected": 0.004694207105785608,
+      "logps/chosen": -1.3602782487869263,
+      "logps/rejected": -1.532902479171753,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3602782487869263,
+      "rewards/margins": 0.1726243793964386,
+      "rewards/rejected": -1.532902479171753,
+      "sft_loss": 1.398123025894165,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 13.463144768403025,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": 0.03331831097602844,
+      "logits/rejected": 0.18258699774742126,
+      "logps/chosen": -1.3355567455291748,
+      "logps/rejected": -1.4422796964645386,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3355567455291748,
+      "rewards/margins": 0.10672305524349213,
+      "rewards/rejected": -1.4422796964645386,
+      "sft_loss": 1.363588571548462,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 4.328631685243004,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": -0.06358928978443146,
+      "logits/rejected": 0.042626697570085526,
+      "logps/chosen": -1.3286917209625244,
+      "logps/rejected": -1.504015564918518,
+      "loss": 0.5522,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3286917209625244,
+      "rewards/margins": 0.17532379925251007,
+      "rewards/rejected": -1.504015564918518,
+      "sft_loss": 1.3659374713897705,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 4.821431945232963,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": 0.039220355451107025,
+      "logits/rejected": 0.10534069687128067,
+      "logps/chosen": -1.4000418186187744,
+      "logps/rejected": -1.5270695686340332,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.4000418186187744,
+      "rewards/margins": 0.12702779471874237,
+      "rewards/rejected": -1.5270695686340332,
+      "sft_loss": 1.401197075843811,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 6.148785201472726,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": 0.07166017591953278,
+      "logits/rejected": 0.1898956000804901,
+      "logps/chosen": -1.3064854145050049,
+      "logps/rejected": -1.4716060161590576,
+      "loss": 0.5241,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3064854145050049,
+      "rewards/margins": 0.1651206910610199,
+      "rewards/rejected": -1.4716060161590576,
+      "sft_loss": 1.3478180170059204,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 6.500178864042233,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": -0.015255662612617016,
+      "logits/rejected": 0.0960661917924881,
+      "logps/chosen": -1.3437426090240479,
+      "logps/rejected": -1.5181114673614502,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3437426090240479,
+      "rewards/margins": 0.1743689477443695,
+      "rewards/rejected": -1.5181114673614502,
+      "sft_loss": 1.3270080089569092,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 4.284856290724194,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": -0.019777098670601845,
+      "logits/rejected": 0.2161232978105545,
+      "logps/chosen": -1.4191901683807373,
+      "logps/rejected": -1.499983310699463,
+      "loss": 0.5951,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.4191901683807373,
+      "rewards/margins": 0.08079305291175842,
+      "rewards/rejected": -1.499983310699463,
+      "sft_loss": 1.4321720600128174,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 12.656078393455692,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": -0.11304919421672821,
+      "logits/rejected": 0.08124849945306778,
+      "logps/chosen": -1.3428949117660522,
+      "logps/rejected": -1.478323221206665,
+      "loss": 0.5509,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3428949117660522,
+      "rewards/margins": 0.13542838394641876,
+      "rewards/rejected": -1.478323221206665,
+      "sft_loss": 1.4283742904663086,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 5.579514201615282,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": -0.09503932297229767,
+      "logits/rejected": -0.008171332068741322,
+      "logps/chosen": -1.2692936658859253,
+      "logps/rejected": -1.471149206161499,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2692936658859253,
+      "rewards/margins": 0.2018553465604782,
+      "rewards/rejected": -1.471149206161499,
+      "sft_loss": 1.2678296566009521,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 7.144522751822136,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": -0.015544240362942219,
+      "logits/rejected": 0.12737944722175598,
+      "logps/chosen": -1.4104164838790894,
+      "logps/rejected": -1.5921103954315186,
+      "loss": 0.5565,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4104164838790894,
+      "rewards/margins": 0.18169382214546204,
+      "rewards/rejected": -1.5921103954315186,
+      "sft_loss": 1.428065299987793,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 6.373414994710634,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": 0.033987972885370255,
+      "logits/rejected": 0.04610954597592354,
+      "logps/chosen": -1.3487951755523682,
+      "logps/rejected": -1.5615766048431396,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3487951755523682,
+      "rewards/margins": 0.21278128027915955,
+      "rewards/rejected": -1.5615766048431396,
+      "sft_loss": 1.366637945175171,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 5.19965274289491,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": 0.014334109611809254,
+      "logits/rejected": 0.14489109814167023,
+      "logps/chosen": -1.3145679235458374,
+      "logps/rejected": -1.390172004699707,
+      "loss": 0.5607,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3145679235458374,
+      "rewards/margins": 0.07560417801141739,
+      "rewards/rejected": -1.390172004699707,
+      "sft_loss": 1.3641953468322754,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 6.443123859185252,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": 0.007931932806968689,
+      "logits/rejected": 0.1458900421857834,
+      "logps/chosen": -1.2717044353485107,
+      "logps/rejected": -1.4361180067062378,
+      "loss": 0.5326,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2717044353485107,
+      "rewards/margins": 0.1644134223461151,
+      "rewards/rejected": -1.4361180067062378,
+      "sft_loss": 1.3242876529693604,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 4.619542183407082,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": -0.017918167635798454,
+      "logits/rejected": 0.1990184336900711,
+      "logps/chosen": -1.3112833499908447,
+      "logps/rejected": -1.3996978998184204,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3112833499908447,
+      "rewards/margins": 0.08841459453105927,
+      "rewards/rejected": -1.3996978998184204,
+      "sft_loss": 1.4118869304656982,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 5.341412538195278,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": -0.03343156352639198,
+      "logits/rejected": 0.04960675165057182,
+      "logps/chosen": -1.3060057163238525,
+      "logps/rejected": -1.5539497137069702,
+      "loss": 0.513,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3060057163238525,
+      "rewards/margins": 0.24794375896453857,
+      "rewards/rejected": -1.5539497137069702,
+      "sft_loss": 1.3633852005004883,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 5.013306968245386,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": 0.07540982961654663,
+      "logits/rejected": 0.23975494503974915,
+      "logps/chosen": -1.357664704322815,
+      "logps/rejected": -1.4599559307098389,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.357664704322815,
+      "rewards/margins": 0.10229126363992691,
+      "rewards/rejected": -1.4599559307098389,
+      "sft_loss": 1.3449851274490356,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 7.835812606614032,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": 0.02607138082385063,
+      "logits/rejected": 0.1844528168439865,
+      "logps/chosen": -1.329110860824585,
+      "logps/rejected": -1.4306621551513672,
+      "loss": 0.5641,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.329110860824585,
+      "rewards/margins": 0.10155129432678223,
+      "rewards/rejected": -1.4306621551513672,
+      "sft_loss": 1.3462426662445068,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 6.033085338974475,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": 0.059251852333545685,
+      "logits/rejected": 0.09896888583898544,
+      "logps/chosen": -1.3339654207229614,
+      "logps/rejected": -1.4830416440963745,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3339654207229614,
+      "rewards/margins": 0.14907608926296234,
+      "rewards/rejected": -1.4830416440963745,
+      "sft_loss": 1.3890200853347778,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 4.424859213314543,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": 0.008425706997513771,
+      "logits/rejected": 0.10140939801931381,
+      "logps/chosen": -1.2267965078353882,
+      "logps/rejected": -1.4236056804656982,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2267965078353882,
+      "rewards/margins": 0.1968090832233429,
+      "rewards/rejected": -1.4236056804656982,
+      "sft_loss": 1.2810438871383667,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 6.692631922724751,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": -0.09820245206356049,
+      "logits/rejected": 0.047392718493938446,
+      "logps/chosen": -1.4009666442871094,
+      "logps/rejected": -1.5001310110092163,
+      "loss": 0.5768,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.4009666442871094,
+      "rewards/margins": 0.09916438907384872,
+      "rewards/rejected": -1.5001310110092163,
+      "sft_loss": 1.385056972503662,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 4.951811140853431,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": -0.07248760014772415,
+      "logits/rejected": 0.10724302381277084,
+      "logps/chosen": -1.336701512336731,
+      "logps/rejected": -1.547623872756958,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.336701512336731,
+      "rewards/margins": 0.21092236042022705,
+      "rewards/rejected": -1.547623872756958,
+      "sft_loss": 1.3786590099334717,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 4.818916350234281,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": 0.05171216279268265,
+      "logits/rejected": 0.14952951669692993,
+      "logps/chosen": -1.2990772724151611,
+      "logps/rejected": -1.4701385498046875,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2990772724151611,
+      "rewards/margins": 0.1710611879825592,
+      "rewards/rejected": -1.4701385498046875,
+      "sft_loss": 1.3045198917388916,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 7.471676922467004,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": -0.012614324688911438,
+      "logits/rejected": 0.1592131406068802,
+      "logps/chosen": -1.371006965637207,
+      "logps/rejected": -1.528407335281372,
+      "loss": 0.5366,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.371006965637207,
+      "rewards/margins": 0.1574004739522934,
+      "rewards/rejected": -1.528407335281372,
+      "sft_loss": 1.3581221103668213,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 5.855309953913271,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": 0.05896524339914322,
+      "logits/rejected": 0.2562909722328186,
+      "logps/chosen": -1.4231863021850586,
+      "logps/rejected": -1.5625927448272705,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4231863021850586,
+      "rewards/margins": 0.13940641283988953,
+      "rewards/rejected": -1.5625927448272705,
+      "sft_loss": 1.4524192810058594,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 5.623027773331276,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": -0.09292992949485779,
+      "logits/rejected": 0.04666275531053543,
+      "logps/chosen": -1.2506234645843506,
+      "logps/rejected": -1.5481257438659668,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.2506234645843506,
+      "rewards/margins": 0.2975021004676819,
+      "rewards/rejected": -1.5481257438659668,
+      "sft_loss": 1.323334813117981,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 5.552946198660787,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": -0.13729050755500793,
+      "logits/rejected": 0.02584907039999962,
+      "logps/chosen": -1.364895224571228,
+      "logps/rejected": -1.5133196115493774,
+      "loss": 0.5632,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.364895224571228,
+      "rewards/margins": 0.14842435717582703,
+      "rewards/rejected": -1.5133196115493774,
+      "sft_loss": 1.3803001642227173,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 5.09955935784875,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": -0.10641257464885712,
+      "logits/rejected": -0.002804142190143466,
+      "logps/chosen": -1.3838285207748413,
+      "logps/rejected": -1.5112088918685913,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3838285207748413,
+      "rewards/margins": 0.12738032639026642,
+      "rewards/rejected": -1.5112088918685913,
+      "sft_loss": 1.4276145696640015,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 8.08936145133186,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": -0.0654737576842308,
+      "logits/rejected": 0.11496500670909882,
+      "logps/chosen": -1.4559099674224854,
+      "logps/rejected": -1.5509315729141235,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.4559099674224854,
+      "rewards/margins": 0.09502150863409042,
+      "rewards/rejected": -1.5509315729141235,
+      "sft_loss": 1.4707705974578857,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 4.860947110964878,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": 0.03588312119245529,
+      "logits/rejected": 0.11505071818828583,
+      "logps/chosen": -1.4208877086639404,
+      "logps/rejected": -1.675723671913147,
+      "loss": 0.569,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4208877086639404,
+      "rewards/margins": 0.2548360228538513,
+      "rewards/rejected": -1.675723671913147,
+      "sft_loss": 1.4088971614837646,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 5.085383768228615,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": -0.0216512531042099,
+      "logits/rejected": 0.14498159289360046,
+      "logps/chosen": -1.360144853591919,
+      "logps/rejected": -1.5747146606445312,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.360144853591919,
+      "rewards/margins": 0.2145698070526123,
+      "rewards/rejected": -1.5747146606445312,
+      "sft_loss": 1.3781875371932983,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 5.0297073385361895,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": 0.009959183633327484,
+      "logits/rejected": 0.14851178228855133,
+      "logps/chosen": -1.3677583932876587,
+      "logps/rejected": -1.519020915031433,
+      "loss": 0.568,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3677583932876587,
+      "rewards/margins": 0.15126249194145203,
+      "rewards/rejected": -1.519020915031433,
+      "sft_loss": 1.408804178237915,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 4.209994488263506,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": -0.07662223279476166,
+      "logits/rejected": -0.00010142624523723498,
+      "logps/chosen": -1.3832899332046509,
+      "logps/rejected": -1.592454195022583,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3832899332046509,
+      "rewards/margins": 0.2091643512248993,
+      "rewards/rejected": -1.592454195022583,
+      "sft_loss": 1.423018217086792,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 5.371155698509368,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": -0.03791964054107666,
+      "logits/rejected": 0.05540591478347778,
+      "logps/chosen": -1.3681023120880127,
+      "logps/rejected": -1.5076701641082764,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3681023120880127,
+      "rewards/margins": 0.13956794142723083,
+      "rewards/rejected": -1.5076701641082764,
+      "sft_loss": 1.3541196584701538,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 6.2165730754532476,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": 0.033802516758441925,
+      "logits/rejected": 0.1595228612422943,
+      "logps/chosen": -1.3133952617645264,
+      "logps/rejected": -1.6211875677108765,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3133952617645264,
+      "rewards/margins": 0.30779242515563965,
+      "rewards/rejected": -1.6211875677108765,
+      "sft_loss": 1.3352806568145752,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 5.954812711931833,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": -0.09634337574243546,
+      "logits/rejected": 0.07502253353595734,
+      "logps/chosen": -1.38030207157135,
+      "logps/rejected": -1.5933905839920044,
+      "loss": 0.55,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.38030207157135,
+      "rewards/margins": 0.2130887806415558,
+      "rewards/rejected": -1.5933905839920044,
+      "sft_loss": 1.3440942764282227,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 4.092097929588904,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": -0.13427576422691345,
+      "logits/rejected": 0.01720806211233139,
+      "logps/chosen": -1.3094775676727295,
+      "logps/rejected": -1.5330116748809814,
+      "loss": 0.5181,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3094775676727295,
+      "rewards/margins": 0.22353395819664001,
+      "rewards/rejected": -1.5330116748809814,
+      "sft_loss": 1.3614782094955444,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 4.773573559503689,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": 0.010130161419510841,
+      "logits/rejected": 0.13926295936107635,
+      "logps/chosen": -1.2965346574783325,
+      "logps/rejected": -1.5689843893051147,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2965346574783325,
+      "rewards/margins": 0.2724498510360718,
+      "rewards/rejected": -1.5689843893051147,
+      "sft_loss": 1.3222274780273438,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 5.362179923085685,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": -0.06429972499608994,
+      "logits/rejected": 0.06651268899440765,
+      "logps/chosen": -1.351266622543335,
+      "logps/rejected": -1.5239372253417969,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.351266622543335,
+      "rewards/margins": 0.1726706176996231,
+      "rewards/rejected": -1.5239372253417969,
+      "sft_loss": 1.3542873859405518,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 6.912654885108542,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": -0.053594689816236496,
+      "logits/rejected": 0.14611445367336273,
+      "logps/chosen": -1.334825873374939,
+      "logps/rejected": -1.5915372371673584,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.334825873374939,
+      "rewards/margins": 0.2567111849784851,
+      "rewards/rejected": -1.5915372371673584,
+      "sft_loss": 1.3713914155960083,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 5.134436625838364,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": -0.02290893718600273,
+      "logits/rejected": 0.0638694167137146,
+      "logps/chosen": -1.3296618461608887,
+      "logps/rejected": -1.4925134181976318,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3296618461608887,
+      "rewards/margins": 0.16285140812397003,
+      "rewards/rejected": -1.4925134181976318,
+      "sft_loss": 1.3362598419189453,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 0.272663950920105,
+      "eval_logits/rejected": 0.3617731034755707,
+      "eval_logps/chosen": -1.3784986734390259,
+      "eval_logps/rejected": -1.6209443807601929,
+      "eval_loss": 0.5318129062652588,
+      "eval_rewards/accuracies": 0.5882789492607117,
+      "eval_rewards/chosen": -1.3784986734390259,
+      "eval_rewards/margins": 0.24244572222232819,
+      "eval_rewards/rejected": -1.6209443807601929,
+      "eval_runtime": 43.3554,
+      "eval_samples_per_second": 31.023,
+      "eval_sft_loss": 1.4044005870819092,
+      "eval_steps_per_second": 7.773,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 6.714069611575347,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": -0.09477131068706512,
+      "logits/rejected": 0.06664792448282242,
+      "logps/chosen": -1.379688024520874,
+      "logps/rejected": -1.6606252193450928,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.379688024520874,
+      "rewards/margins": 0.2809372842311859,
+      "rewards/rejected": -1.6606252193450928,
+      "sft_loss": 1.4098918437957764,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 6.023653869387464,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": 0.019546739757061005,
+      "logits/rejected": 0.0949871689081192,
+      "logps/chosen": -1.3077045679092407,
+      "logps/rejected": -1.5455154180526733,
+      "loss": 0.5199,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3077045679092407,
+      "rewards/margins": 0.23781093955039978,
+      "rewards/rejected": -1.5455154180526733,
+      "sft_loss": 1.3323251008987427,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 7.590492601575631,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": 0.01001892052590847,
+      "logits/rejected": 0.1464158594608307,
+      "logps/chosen": -1.3059206008911133,
+      "logps/rejected": -1.6408694982528687,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3059206008911133,
+      "rewards/margins": 0.3349488377571106,
+      "rewards/rejected": -1.6408694982528687,
+      "sft_loss": 1.3800963163375854,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 6.846753860809149,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": 0.02080114185810089,
+      "logits/rejected": 0.07514943182468414,
+      "logps/chosen": -1.390001654624939,
+      "logps/rejected": -1.6592248678207397,
+      "loss": 0.5336,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.390001654624939,
+      "rewards/margins": 0.26922309398651123,
+      "rewards/rejected": -1.6592248678207397,
+      "sft_loss": 1.4036567211151123,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 13.520316774007492,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": -0.04507501795887947,
+      "logits/rejected": 0.08645032346248627,
+      "logps/chosen": -1.315307855606079,
+      "logps/rejected": -1.5431115627288818,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.315307855606079,
+      "rewards/margins": 0.22780366241931915,
+      "rewards/rejected": -1.5431115627288818,
+      "sft_loss": 1.4078117609024048,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 6.587293554273986,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": 0.06493745744228363,
+      "logits/rejected": 0.23367440700531006,
+      "logps/chosen": -1.3996050357818604,
+      "logps/rejected": -1.5623871088027954,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3996050357818604,
+      "rewards/margins": 0.16278214752674103,
+      "rewards/rejected": -1.5623871088027954,
+      "sft_loss": 1.3790435791015625,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 6.37550903604283,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": 0.012400135397911072,
+      "logits/rejected": 0.15300646424293518,
+      "logps/chosen": -1.32003653049469,
+      "logps/rejected": -1.5273606777191162,
+      "loss": 0.5311,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.32003653049469,
+      "rewards/margins": 0.20732417702674866,
+      "rewards/rejected": -1.5273606777191162,
+      "sft_loss": 1.3879741430282593,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 5.143220604120998,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": 0.015577117912471294,
+      "logits/rejected": 0.1794794499874115,
+      "logps/chosen": -1.365195870399475,
+      "logps/rejected": -1.6385002136230469,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.365195870399475,
+      "rewards/margins": 0.27330437302589417,
+      "rewards/rejected": -1.6385002136230469,
+      "sft_loss": 1.4092786312103271,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 3.06536112732823,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": 0.06246393918991089,
+      "logits/rejected": 0.13810136914253235,
+      "logps/chosen": -1.3615779876708984,
+      "logps/rejected": -1.5043030977249146,
+      "loss": 0.5668,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3615779876708984,
+      "rewards/margins": 0.14272502064704895,
+      "rewards/rejected": -1.5043030977249146,
+      "sft_loss": 1.4312576055526733,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 6.6171418934639314,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": 0.017917241901159286,
+      "logits/rejected": 0.09261620789766312,
+      "logps/chosen": -1.3315399885177612,
+      "logps/rejected": -1.5422439575195312,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3315399885177612,
+      "rewards/margins": 0.21070384979248047,
+      "rewards/rejected": -1.5422439575195312,
+      "sft_loss": 1.3558510541915894,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 7.754700261996109,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": -0.001633057021535933,
+      "logits/rejected": 0.1511707454919815,
+      "logps/chosen": -1.394895076751709,
+      "logps/rejected": -1.6898698806762695,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.394895076751709,
+      "rewards/margins": 0.29497483372688293,
+      "rewards/rejected": -1.6898698806762695,
+      "sft_loss": 1.4036824703216553,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 6.417177809723641,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": -0.06009018421173096,
+      "logits/rejected": 0.10860785096883774,
+      "logps/chosen": -1.4335905313491821,
+      "logps/rejected": -1.7497847080230713,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4335905313491821,
+      "rewards/margins": 0.3161943256855011,
+      "rewards/rejected": -1.7497847080230713,
+      "sft_loss": 1.4489614963531494,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 8.61348044658711,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": 0.023756619542837143,
+      "logits/rejected": 0.12181626260280609,
+      "logps/chosen": -1.3452613353729248,
+      "logps/rejected": -1.6940526962280273,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3452613353729248,
+      "rewards/margins": 0.3487912118434906,
+      "rewards/rejected": -1.6940526962280273,
+      "sft_loss": 1.3578007221221924,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 5.418686768484927,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": -0.07046753168106079,
+      "logits/rejected": 0.2019074410200119,
+      "logps/chosen": -1.3916847705841064,
+      "logps/rejected": -1.6394716501235962,
+      "loss": 0.5432,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3916847705841064,
+      "rewards/margins": 0.2477869987487793,
+      "rewards/rejected": -1.6394716501235962,
+      "sft_loss": 1.459414005279541,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 6.772815715405938,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": 0.04510253667831421,
+      "logits/rejected": 0.12748414278030396,
+      "logps/chosen": -1.4449238777160645,
+      "logps/rejected": -1.6484653949737549,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4449238777160645,
+      "rewards/margins": 0.20354151725769043,
+      "rewards/rejected": -1.6484653949737549,
+      "sft_loss": 1.4168418645858765,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 5.992740242144888,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": 0.03951137140393257,
+      "logits/rejected": 0.20961646735668182,
+      "logps/chosen": -1.4726355075836182,
+      "logps/rejected": -1.7111984491348267,
+      "loss": 0.5574,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4726355075836182,
+      "rewards/margins": 0.23856297135353088,
+      "rewards/rejected": -1.7111984491348267,
+      "sft_loss": 1.3962585926055908,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 5.702059239685596,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": -0.026392418891191483,
+      "logits/rejected": 0.058989159762859344,
+      "logps/chosen": -1.3647552728652954,
+      "logps/rejected": -1.648855209350586,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3647552728652954,
+      "rewards/margins": 0.2841000258922577,
+      "rewards/rejected": -1.648855209350586,
+      "sft_loss": 1.4022343158721924,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 4.659252230628318,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": -0.13175614178180695,
+      "logits/rejected": 0.0019244104623794556,
+      "logps/chosen": -1.4099591970443726,
+      "logps/rejected": -1.6276271343231201,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4099591970443726,
+      "rewards/margins": 0.21766802668571472,
+      "rewards/rejected": -1.6276271343231201,
+      "sft_loss": 1.463881492614746,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 4.770433623064993,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": 0.07327640056610107,
+      "logits/rejected": 0.19160141050815582,
+      "logps/chosen": -1.283439040184021,
+      "logps/rejected": -1.4736160039901733,
+      "loss": 0.5258,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.283439040184021,
+      "rewards/margins": 0.19017700850963593,
+      "rewards/rejected": -1.4736160039901733,
+      "sft_loss": 1.3432611227035522,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 5.188465336691229,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": -0.06001676991581917,
+      "logits/rejected": 0.012325121089816093,
+      "logps/chosen": -1.284280776977539,
+      "logps/rejected": -1.5858594179153442,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.284280776977539,
+      "rewards/margins": 0.30157846212387085,
+      "rewards/rejected": -1.5858594179153442,
+      "sft_loss": 1.3404700756072998,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 6.435494847996555,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": 0.030720632523298264,
+      "logits/rejected": 0.22497057914733887,
+      "logps/chosen": -1.379372477531433,
+      "logps/rejected": -1.6559817790985107,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.379372477531433,
+      "rewards/margins": 0.2766092121601105,
+      "rewards/rejected": -1.6559817790985107,
+      "sft_loss": 1.4437005519866943,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 3.2066285057340056,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": 0.007974756881594658,
+      "logits/rejected": 0.11983414739370346,
+      "logps/chosen": -1.4005982875823975,
+      "logps/rejected": -1.572317123413086,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4005982875823975,
+      "rewards/margins": 0.1717187911272049,
+      "rewards/rejected": -1.572317123413086,
+      "sft_loss": 1.3948160409927368,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 5.693222226682654,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": 0.007499815430492163,
+      "logits/rejected": 0.19532763957977295,
+      "logps/chosen": -1.3502609729766846,
+      "logps/rejected": -1.559653639793396,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3502609729766846,
+      "rewards/margins": 0.20939283072948456,
+      "rewards/rejected": -1.559653639793396,
+      "sft_loss": 1.3668898344039917,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 6.159470735695498,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": 0.14137479662895203,
+      "logits/rejected": 0.287406325340271,
+      "logps/chosen": -1.2880275249481201,
+      "logps/rejected": -1.5608675479888916,
+      "loss": 0.5292,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2880275249481201,
+      "rewards/margins": 0.2728400230407715,
+      "rewards/rejected": -1.5608675479888916,
+      "sft_loss": 1.352113962173462,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 6.255987887486548,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": -0.013386614620685577,
+      "logits/rejected": 0.1623094230890274,
+      "logps/chosen": -1.4105921983718872,
+      "logps/rejected": -1.5581471920013428,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4105921983718872,
+      "rewards/margins": 0.14755511283874512,
+      "rewards/rejected": -1.5581471920013428,
+      "sft_loss": 1.4051482677459717,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 8.388676102005673,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": -0.0396624319255352,
+      "logits/rejected": 0.07985645532608032,
+      "logps/chosen": -1.3447215557098389,
+      "logps/rejected": -1.6878904104232788,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3447215557098389,
+      "rewards/margins": 0.34316885471343994,
+      "rewards/rejected": -1.6878904104232788,
+      "sft_loss": 1.3712944984436035,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 4.908854695224659,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": 0.05237286165356636,
+      "logits/rejected": 0.09974393993616104,
+      "logps/chosen": -1.3034316301345825,
+      "logps/rejected": -1.5620901584625244,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3034316301345825,
+      "rewards/margins": 0.25865834951400757,
+      "rewards/rejected": -1.5620901584625244,
+      "sft_loss": 1.3588597774505615,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 6.89034964552559,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": 0.006044765003025532,
+      "logits/rejected": 0.12427693605422974,
+      "logps/chosen": -1.367592215538025,
+      "logps/rejected": -1.5376479625701904,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.367592215538025,
+      "rewards/margins": 0.17005568742752075,
+      "rewards/rejected": -1.5376479625701904,
+      "sft_loss": 1.4046542644500732,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 5.965890416950193,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": -0.013211125507950783,
+      "logits/rejected": 0.12869153916835785,
+      "logps/chosen": -1.353560209274292,
+      "logps/rejected": -1.6667423248291016,
+      "loss": 0.5,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.353560209274292,
+      "rewards/margins": 0.31318214535713196,
+      "rewards/rejected": -1.6667423248291016,
+      "sft_loss": 1.361161708831787,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 5.910356268986856,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": -0.02266688272356987,
+      "logits/rejected": 0.1527927666902542,
+      "logps/chosen": -1.4592188596725464,
+      "logps/rejected": -1.6313902139663696,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.4592188596725464,
+      "rewards/margins": 0.17217124998569489,
+      "rewards/rejected": -1.6313902139663696,
+      "sft_loss": 1.4636166095733643,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 5.828938279226205,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": 0.025441814213991165,
+      "logits/rejected": 0.2594681978225708,
+      "logps/chosen": -1.4633724689483643,
+      "logps/rejected": -1.6635366678237915,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4633724689483643,
+      "rewards/margins": 0.20016424357891083,
+      "rewards/rejected": -1.6635366678237915,
+      "sft_loss": 1.5087789297103882,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 6.916444373102311,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": -0.019559945911169052,
+      "logits/rejected": 0.13119149208068848,
+      "logps/chosen": -1.3910858631134033,
+      "logps/rejected": -1.5714839696884155,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3910858631134033,
+      "rewards/margins": 0.1803981512784958,
+      "rewards/rejected": -1.5714839696884155,
+      "sft_loss": 1.4235373735427856,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 5.783815333575644,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": 0.06887931376695633,
+      "logits/rejected": 0.0669599249958992,
+      "logps/chosen": -1.4362841844558716,
+      "logps/rejected": -1.7442407608032227,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4362841844558716,
+      "rewards/margins": 0.3079567551612854,
+      "rewards/rejected": -1.7442407608032227,
+      "sft_loss": 1.4661349058151245,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 5.002807863509631,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": 0.09101974219083786,
+      "logits/rejected": 0.10050985962152481,
+      "logps/chosen": -1.3150116205215454,
+      "logps/rejected": -1.5524133443832397,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3150116205215454,
+      "rewards/margins": 0.23740187287330627,
+      "rewards/rejected": -1.5524133443832397,
+      "sft_loss": 1.3296570777893066,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 5.278186807526357,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": -0.05638844892382622,
+      "logits/rejected": 0.14079150557518005,
+      "logps/chosen": -1.3863670825958252,
+      "logps/rejected": -1.5993766784667969,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3863670825958252,
+      "rewards/margins": 0.21300962567329407,
+      "rewards/rejected": -1.5993766784667969,
+      "sft_loss": 1.4488608837127686,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 7.842449217425358,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": -0.04868556931614876,
+      "logits/rejected": 0.15271759033203125,
+      "logps/chosen": -1.4653922319412231,
+      "logps/rejected": -1.6876140832901,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4653922319412231,
+      "rewards/margins": 0.22222192585468292,
+      "rewards/rejected": -1.6876140832901,
+      "sft_loss": 1.4675097465515137,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 6.6791345392995725,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": 0.03832697868347168,
+      "logits/rejected": 0.23097598552703857,
+      "logps/chosen": -1.4058834314346313,
+      "logps/rejected": -1.648074746131897,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4058834314346313,
+      "rewards/margins": 0.24219119548797607,
+      "rewards/rejected": -1.648074746131897,
+      "sft_loss": 1.4827638864517212,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 4.400687968516655,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": 0.07663445919752121,
+      "logits/rejected": 0.11500799655914307,
+      "logps/chosen": -1.2576382160186768,
+      "logps/rejected": -1.5772205591201782,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2576382160186768,
+      "rewards/margins": 0.3195821940898895,
+      "rewards/rejected": -1.5772205591201782,
+      "sft_loss": 1.3516136407852173,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 5.496637559141165,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": 0.12294967472553253,
+      "logits/rejected": 0.209446981549263,
+      "logps/chosen": -1.3757258653640747,
+      "logps/rejected": -1.53915274143219,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3757258653640747,
+      "rewards/margins": 0.16342684626579285,
+      "rewards/rejected": -1.53915274143219,
+      "sft_loss": 1.4503815174102783,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 7.084586699245085,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": 0.11083235591650009,
+      "logits/rejected": 0.2419196367263794,
+      "logps/chosen": -1.403327465057373,
+      "logps/rejected": -1.692497968673706,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.403327465057373,
+      "rewards/margins": 0.2891707420349121,
+      "rewards/rejected": -1.692497968673706,
+      "sft_loss": 1.4336564540863037,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 8.470327477689404,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": 0.06831010431051254,
+      "logits/rejected": 0.24153780937194824,
+      "logps/chosen": -1.4360759258270264,
+      "logps/rejected": -1.7128865718841553,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4360759258270264,
+      "rewards/margins": 0.2768106460571289,
+      "rewards/rejected": -1.7128865718841553,
+      "sft_loss": 1.484975814819336,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 5.585928773503728,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": 0.017606690526008606,
+      "logits/rejected": 0.2122047394514084,
+      "logps/chosen": -1.2979730367660522,
+      "logps/rejected": -1.6680152416229248,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2979730367660522,
+      "rewards/margins": 0.37004226446151733,
+      "rewards/rejected": -1.6680152416229248,
+      "sft_loss": 1.3362150192260742,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 6.0789178538079325,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": 0.031071409583091736,
+      "logits/rejected": 0.21970124542713165,
+      "logps/chosen": -1.351664662361145,
+      "logps/rejected": -1.6419140100479126,
+      "loss": 0.514,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.351664662361145,
+      "rewards/margins": 0.29024919867515564,
+      "rewards/rejected": -1.6419140100479126,
+      "sft_loss": 1.4079734086990356,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 6.839970819560228,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": 0.038407161831855774,
+      "logits/rejected": 0.1905667930841446,
+      "logps/chosen": -1.4028685092926025,
+      "logps/rejected": -1.7034860849380493,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4028685092926025,
+      "rewards/margins": 0.300617516040802,
+      "rewards/rejected": -1.7034860849380493,
+      "sft_loss": 1.4510376453399658,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 6.261514936310619,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": -0.0038838565815240145,
+      "logits/rejected": 0.12509891390800476,
+      "logps/chosen": -1.3250789642333984,
+      "logps/rejected": -1.6913788318634033,
+      "loss": 0.504,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3250789642333984,
+      "rewards/margins": 0.3662997782230377,
+      "rewards/rejected": -1.6913788318634033,
+      "sft_loss": 1.3589133024215698,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 7.532636921841018,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": 0.021292533725500107,
+      "logits/rejected": 0.11308477818965912,
+      "logps/chosen": -1.3988656997680664,
+      "logps/rejected": -1.6569464206695557,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3988656997680664,
+      "rewards/margins": 0.2580806612968445,
+      "rewards/rejected": -1.6569464206695557,
+      "sft_loss": 1.4494415521621704,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 7.267928662402311,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": -0.0075324522331357,
+      "logits/rejected": 0.11378462612628937,
+      "logps/chosen": -1.455899953842163,
+      "logps/rejected": -1.6032978296279907,
+      "loss": 0.5654,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.455899953842163,
+      "rewards/margins": 0.14739762246608734,
+      "rewards/rejected": -1.6032978296279907,
+      "sft_loss": 1.462633490562439,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 9.270113397343684,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": 0.03945959731936455,
+      "logits/rejected": 0.20420022308826447,
+      "logps/chosen": -1.355939269065857,
+      "logps/rejected": -1.6491836309432983,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.355939269065857,
+      "rewards/margins": 0.2932443618774414,
+      "rewards/rejected": -1.6491836309432983,
+      "sft_loss": 1.4178965091705322,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 3.9428774371292308,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": -0.029956543818116188,
+      "logits/rejected": 0.1199512928724289,
+      "logps/chosen": -1.3319822549819946,
+      "logps/rejected": -1.6581604480743408,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3319822549819946,
+      "rewards/margins": 0.32617828249931335,
+      "rewards/rejected": -1.6581604480743408,
+      "sft_loss": 1.4374191761016846,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 8.900796310027207,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": -0.08117417246103287,
+      "logits/rejected": 0.0267164446413517,
+      "logps/chosen": -1.424697756767273,
+      "logps/rejected": -1.7513818740844727,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.424697756767273,
+      "rewards/margins": 0.32668399810791016,
+      "rewards/rejected": -1.7513818740844727,
+      "sft_loss": 1.4675133228302002,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 8.31046841086775,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": 0.008648221381008625,
+      "logits/rejected": 0.09285394847393036,
+      "logps/chosen": -1.428062081336975,
+      "logps/rejected": -1.715243935585022,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.428062081336975,
+      "rewards/margins": 0.2871818542480469,
+      "rewards/rejected": -1.715243935585022,
+      "sft_loss": 1.459547758102417,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 8.420046513363681,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": 0.03144007921218872,
+      "logits/rejected": 0.2124161720275879,
+      "logps/chosen": -1.5106102228164673,
+      "logps/rejected": -1.763193130493164,
+      "loss": 0.5501,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5106102228164673,
+      "rewards/margins": 0.25258293747901917,
+      "rewards/rejected": -1.763193130493164,
+      "sft_loss": 1.495023488998413,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 3.7418036331803304,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": 0.047572363168001175,
+      "logits/rejected": 0.18473589420318604,
+      "logps/chosen": -1.4035441875457764,
+      "logps/rejected": -1.80259108543396,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4035441875457764,
+      "rewards/margins": 0.3990468978881836,
+      "rewards/rejected": -1.80259108543396,
+      "sft_loss": 1.4959805011749268,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 5.7848942867266695,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": -0.08872415125370026,
+      "logits/rejected": 0.15845569968223572,
+      "logps/chosen": -1.456129789352417,
+      "logps/rejected": -1.7990448474884033,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.456129789352417,
+      "rewards/margins": 0.34291499853134155,
+      "rewards/rejected": -1.7990448474884033,
+      "sft_loss": 1.4630060195922852,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 4.524211282548457,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": -0.018144235014915466,
+      "logits/rejected": 0.11269550025463104,
+      "logps/chosen": -1.3974212408065796,
+      "logps/rejected": -1.774336576461792,
+      "loss": 0.5062,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3974212408065796,
+      "rewards/margins": 0.3769153952598572,
+      "rewards/rejected": -1.774336576461792,
+      "sft_loss": 1.4539904594421387,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 6.618978905317209,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": -0.06820371747016907,
+      "logits/rejected": 0.0417419895529747,
+      "logps/chosen": -1.5021356344223022,
+      "logps/rejected": -1.7642343044281006,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5021356344223022,
+      "rewards/margins": 0.26209884881973267,
+      "rewards/rejected": -1.7642343044281006,
+      "sft_loss": 1.5417989492416382,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 8.575313314221772,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": -0.010738177224993706,
+      "logits/rejected": 0.15471696853637695,
+      "logps/chosen": -1.4406782388687134,
+      "logps/rejected": -1.554862141609192,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.4406782388687134,
+      "rewards/margins": 0.11418372392654419,
+      "rewards/rejected": -1.554862141609192,
+      "sft_loss": 1.4563452005386353,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 6.240052893582782,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": -0.0038662850856781006,
+      "logits/rejected": 0.10398884862661362,
+      "logps/chosen": -1.40817129611969,
+      "logps/rejected": -1.5353500843048096,
+      "loss": 0.571,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.40817129611969,
+      "rewards/margins": 0.1271788775920868,
+      "rewards/rejected": -1.5353500843048096,
+      "sft_loss": 1.4028632640838623,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 5.2462033549546865,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": -0.13482658565044403,
+      "logits/rejected": -0.0035198740661144257,
+      "logps/chosen": -1.3518860340118408,
+      "logps/rejected": -1.6204659938812256,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3518860340118408,
+      "rewards/margins": 0.26857990026474,
+      "rewards/rejected": -1.6204659938812256,
+      "sft_loss": 1.4049688577651978,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 5.863844446116353,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": 0.015030590817332268,
+      "logits/rejected": 0.11544246971607208,
+      "logps/chosen": -1.2636845111846924,
+      "logps/rejected": -1.7326905727386475,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2636845111846924,
+      "rewards/margins": 0.46900612115859985,
+      "rewards/rejected": -1.7326905727386475,
+      "sft_loss": 1.3195103406906128,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 4.594870990121541,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": -0.06224752590060234,
+      "logits/rejected": 0.020744603127241135,
+      "logps/chosen": -1.381974220275879,
+      "logps/rejected": -1.5955941677093506,
+      "loss": 0.5413,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.381974220275879,
+      "rewards/margins": 0.21361997723579407,
+      "rewards/rejected": -1.5955941677093506,
+      "sft_loss": 1.3867851495742798,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 6.092708307677853,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": -0.04417000710964203,
+      "logits/rejected": 0.07652486860752106,
+      "logps/chosen": -1.3594367504119873,
+      "logps/rejected": -1.519745111465454,
+      "loss": 0.5404,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3594367504119873,
+      "rewards/margins": 0.16030827164649963,
+      "rewards/rejected": -1.519745111465454,
+      "sft_loss": 1.3841520547866821,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 10.943720385690531,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": 0.029814088717103004,
+      "logits/rejected": 0.1491873860359192,
+      "logps/chosen": -1.2818458080291748,
+      "logps/rejected": -1.5984141826629639,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2818458080291748,
+      "rewards/margins": 0.31656843423843384,
+      "rewards/rejected": -1.5984141826629639,
+      "sft_loss": 1.2908055782318115,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 5.5118942012596515,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": -0.15234415233135223,
+      "logits/rejected": -0.04955081641674042,
+      "logps/chosen": -1.3449537754058838,
+      "logps/rejected": -1.6491496562957764,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3449537754058838,
+      "rewards/margins": 0.304195761680603,
+      "rewards/rejected": -1.6491496562957764,
+      "sft_loss": 1.3979218006134033,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 8.021170292513808,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": -0.15768754482269287,
+      "logits/rejected": 0.024920865893363953,
+      "logps/chosen": -1.4138802289962769,
+      "logps/rejected": -1.7606861591339111,
+      "loss": 0.5116,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4138802289962769,
+      "rewards/margins": 0.3468059003353119,
+      "rewards/rejected": -1.7606861591339111,
+      "sft_loss": 1.4192440509796143,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 6.295529086087212,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": 0.013290470466017723,
+      "logits/rejected": 0.06877493858337402,
+      "logps/chosen": -1.4313960075378418,
+      "logps/rejected": -1.7183454036712646,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4313960075378418,
+      "rewards/margins": 0.2869493067264557,
+      "rewards/rejected": -1.7183454036712646,
+      "sft_loss": 1.3810927867889404,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 8.859671073714802,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": -0.06083168461918831,
+      "logits/rejected": 0.09201276302337646,
+      "logps/chosen": -1.5324196815490723,
+      "logps/rejected": -1.8598239421844482,
+      "loss": 0.5551,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5324196815490723,
+      "rewards/margins": 0.3274044394493103,
+      "rewards/rejected": -1.8598239421844482,
+      "sft_loss": 1.4262425899505615,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 7.030273446229885,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": -0.06408004462718964,
+      "logits/rejected": -0.016680490225553513,
+      "logps/chosen": -1.4756966829299927,
+      "logps/rejected": -1.6183178424835205,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4756966829299927,
+      "rewards/margins": 0.1426212340593338,
+      "rewards/rejected": -1.6183178424835205,
+      "sft_loss": 1.5306628942489624,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 5.8221650590700404,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": -0.18864266574382782,
+      "logits/rejected": -0.0647595077753067,
+      "logps/chosen": -1.4422852993011475,
+      "logps/rejected": -1.7732452154159546,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4422852993011475,
+      "rewards/margins": 0.330960214138031,
+      "rewards/rejected": -1.7732452154159546,
+      "sft_loss": 1.5205518007278442,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 5.871581315934256,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": -0.16291369497776031,
+      "logits/rejected": -0.028021007776260376,
+      "logps/chosen": -1.3359414339065552,
+      "logps/rejected": -1.6861404180526733,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.3359414339065552,
+      "rewards/margins": 0.3501989245414734,
+      "rewards/rejected": -1.6861404180526733,
+      "sft_loss": 1.3437201976776123,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 5.387313145530907,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": -0.07084006071090698,
+      "logits/rejected": 0.022573301568627357,
+      "logps/chosen": -1.4162758588790894,
+      "logps/rejected": -1.6274665594100952,
+      "loss": 0.546,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4162758588790894,
+      "rewards/margins": 0.21119055151939392,
+      "rewards/rejected": -1.6274665594100952,
+      "sft_loss": 1.4579969644546509,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 6.915753400263389,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": -0.010980304330587387,
+      "logits/rejected": 0.08265860378742218,
+      "logps/chosen": -1.3437680006027222,
+      "logps/rejected": -1.6253420114517212,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.3437680006027222,
+      "rewards/margins": 0.28157392144203186,
+      "rewards/rejected": -1.6253420114517212,
+      "sft_loss": 1.4295752048492432,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 4.3569418482421955,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": -0.06969741731882095,
+      "logits/rejected": 0.010083474218845367,
+      "logps/chosen": -1.3329765796661377,
+      "logps/rejected": -1.5761654376983643,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.3329765796661377,
+      "rewards/margins": 0.2431887835264206,
+      "rewards/rejected": -1.5761654376983643,
+      "sft_loss": 1.3583405017852783,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 4.473590718113074,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": -0.08624277263879776,
+      "logits/rejected": 0.07240621745586395,
+      "logps/chosen": -1.5664074420928955,
+      "logps/rejected": -1.7688283920288086,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5664074420928955,
+      "rewards/margins": 0.20242106914520264,
+      "rewards/rejected": -1.7688283920288086,
+      "sft_loss": 1.532112717628479,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 4.114132484032189,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": -0.07057038694620132,
+      "logits/rejected": 0.10959811508655548,
+      "logps/chosen": -1.3659435510635376,
+      "logps/rejected": -1.6666004657745361,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3659435510635376,
+      "rewards/margins": 0.30065685510635376,
+      "rewards/rejected": -1.6666004657745361,
+      "sft_loss": 1.4245878458023071,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 5.164074151299386,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": -0.08701647818088531,
+      "logits/rejected": 0.05283905193209648,
+      "logps/chosen": -1.4877811670303345,
+      "logps/rejected": -1.8170543909072876,
+      "loss": 0.5181,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4877811670303345,
+      "rewards/margins": 0.3292733430862427,
+      "rewards/rejected": -1.8170543909072876,
+      "sft_loss": 1.4720274209976196,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 7.34774728696768,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": -0.017324324697256088,
+      "logits/rejected": 0.12090086936950684,
+      "logps/chosen": -1.5208476781845093,
+      "logps/rejected": -1.7934768199920654,
+      "loss": 0.5548,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5208476781845093,
+      "rewards/margins": 0.2726292014122009,
+      "rewards/rejected": -1.7934768199920654,
+      "sft_loss": 1.473417043685913,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 6.258376100266322,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": -0.0985368937253952,
+      "logits/rejected": 0.04786267876625061,
+      "logps/chosen": -1.4638845920562744,
+      "logps/rejected": -1.8589398860931396,
+      "loss": 0.4907,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4638845920562744,
+      "rewards/margins": 0.39505526423454285,
+      "rewards/rejected": -1.8589398860931396,
+      "sft_loss": 1.4805598258972168,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 4.5357941902003915,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": -0.04397290572524071,
+      "logits/rejected": 0.07348016649484634,
+      "logps/chosen": -1.344813346862793,
+      "logps/rejected": -1.7454315423965454,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.344813346862793,
+      "rewards/margins": 0.4006180763244629,
+      "rewards/rejected": -1.7454315423965454,
+      "sft_loss": 1.427937626838684,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 9.259125217654567,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": -0.11931729316711426,
+      "logits/rejected": -0.050427474081516266,
+      "logps/chosen": -1.4302603006362915,
+      "logps/rejected": -1.7228256464004517,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4302603006362915,
+      "rewards/margins": 0.2925655245780945,
+      "rewards/rejected": -1.7228256464004517,
+      "sft_loss": 1.433077096939087,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": 0.2312067151069641,
+      "eval_logits/rejected": 0.32195788621902466,
+      "eval_logps/chosen": -1.439756155014038,
+      "eval_logps/rejected": -1.789048194885254,
+      "eval_loss": 0.5156100392341614,
+      "eval_rewards/accuracies": 0.6186943650245667,
+      "eval_rewards/chosen": -1.439756155014038,
+      "eval_rewards/margins": 0.349292129278183,
+      "eval_rewards/rejected": -1.789048194885254,
+      "eval_runtime": 43.2063,
+      "eval_samples_per_second": 31.13,
+      "eval_sft_loss": 1.4648340940475464,
+      "eval_steps_per_second": 7.8,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 7.497624604633386,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": -0.2047913521528244,
+      "logits/rejected": -0.028477761894464493,
+      "logps/chosen": -1.4075069427490234,
+      "logps/rejected": -1.695677399635315,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4075069427490234,
+      "rewards/margins": 0.28817063570022583,
+      "rewards/rejected": -1.695677399635315,
+      "sft_loss": 1.457287073135376,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 5.828011221905981,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": -0.10706796497106552,
+      "logits/rejected": 0.09093932807445526,
+      "logps/chosen": -1.3917760848999023,
+      "logps/rejected": -1.6692960262298584,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3917760848999023,
+      "rewards/margins": 0.2775200605392456,
+      "rewards/rejected": -1.6692960262298584,
+      "sft_loss": 1.4143939018249512,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 4.494440416665394,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": -0.13113786280155182,
+      "logits/rejected": -0.008872026577591896,
+      "logps/chosen": -1.412584900856018,
+      "logps/rejected": -1.5937589406967163,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.412584900856018,
+      "rewards/margins": 0.1811741143465042,
+      "rewards/rejected": -1.5937589406967163,
+      "sft_loss": 1.4540883302688599,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 6.339627030901809,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": 0.03890204429626465,
+      "logits/rejected": 0.1465032696723938,
+      "logps/chosen": -1.3235418796539307,
+      "logps/rejected": -1.5791696310043335,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3235418796539307,
+      "rewards/margins": 0.25562790036201477,
+      "rewards/rejected": -1.5791696310043335,
+      "sft_loss": 1.3354781866073608,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 5.280026773959482,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": -0.15729853510856628,
+      "logits/rejected": 0.05795856565237045,
+      "logps/chosen": -1.4452846050262451,
+      "logps/rejected": -1.6697638034820557,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4452846050262451,
+      "rewards/margins": 0.224479079246521,
+      "rewards/rejected": -1.6697638034820557,
+      "sft_loss": 1.487400770187378,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 6.4958026268078966,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": -0.054301343858242035,
+      "logits/rejected": 0.048134349286556244,
+      "logps/chosen": -1.3963154554367065,
+      "logps/rejected": -1.656686782836914,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3963154554367065,
+      "rewards/margins": 0.2603713870048523,
+      "rewards/rejected": -1.656686782836914,
+      "sft_loss": 1.3829001188278198,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 4.727866949980633,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": -0.01401115395128727,
+      "logits/rejected": 0.12580223381519318,
+      "logps/chosen": -1.4165058135986328,
+      "logps/rejected": -1.6185691356658936,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4165058135986328,
+      "rewards/margins": 0.20206335186958313,
+      "rewards/rejected": -1.6185691356658936,
+      "sft_loss": 1.5026344060897827,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 5.596557785835436,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": -0.16433796286582947,
+      "logits/rejected": -0.03051048144698143,
+      "logps/chosen": -1.3875510692596436,
+      "logps/rejected": -1.6686878204345703,
+      "loss": 0.5052,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3875510692596436,
+      "rewards/margins": 0.2811369299888611,
+      "rewards/rejected": -1.6686878204345703,
+      "sft_loss": 1.4683420658111572,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 5.21063206823352,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": -0.06284203380346298,
+      "logits/rejected": 0.12157057225704193,
+      "logps/chosen": -1.4473823308944702,
+      "logps/rejected": -1.6761335134506226,
+      "loss": 0.5671,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4473823308944702,
+      "rewards/margins": 0.22875113785266876,
+      "rewards/rejected": -1.6761335134506226,
+      "sft_loss": 1.5018178224563599,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 8.033877889431182,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": -0.06265360862016678,
+      "logits/rejected": 0.14023162424564362,
+      "logps/chosen": -1.5192325115203857,
+      "logps/rejected": -1.736997365951538,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.5192325115203857,
+      "rewards/margins": 0.21776482462882996,
+      "rewards/rejected": -1.736997365951538,
+      "sft_loss": 1.5233339071273804,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 7.9555308238631905,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": -0.060469646006822586,
+      "logits/rejected": 0.07171236723661423,
+      "logps/chosen": -1.4047214984893799,
+      "logps/rejected": -1.686340093612671,
+      "loss": 0.5278,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4047214984893799,
+      "rewards/margins": 0.28161847591400146,
+      "rewards/rejected": -1.686340093612671,
+      "sft_loss": 1.4950298070907593,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 4.922792255561007,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": -0.09880800545215607,
+      "logits/rejected": 0.03451662138104439,
+      "logps/chosen": -1.399361491203308,
+      "logps/rejected": -1.7778685092926025,
+      "loss": 0.5055,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.399361491203308,
+      "rewards/margins": 0.3785068094730377,
+      "rewards/rejected": -1.7778685092926025,
+      "sft_loss": 1.468930959701538,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 6.19723798522441,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": 0.0034088001120835543,
+      "logits/rejected": 0.1572073996067047,
+      "logps/chosen": -1.4213447570800781,
+      "logps/rejected": -1.7516653537750244,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4213447570800781,
+      "rewards/margins": 0.33032044768333435,
+      "rewards/rejected": -1.7516653537750244,
+      "sft_loss": 1.4661481380462646,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 6.050036047925458,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": -0.09415547549724579,
+      "logits/rejected": 0.01310119591653347,
+      "logps/chosen": -1.4122450351715088,
+      "logps/rejected": -1.7994779348373413,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4122450351715088,
+      "rewards/margins": 0.3872327208518982,
+      "rewards/rejected": -1.7994779348373413,
+      "sft_loss": 1.4726154804229736,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 7.399312373987854,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": -0.049418993294239044,
+      "logits/rejected": 0.0669780820608139,
+      "logps/chosen": -1.4792741537094116,
+      "logps/rejected": -1.706899642944336,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4792741537094116,
+      "rewards/margins": 0.22762539982795715,
+      "rewards/rejected": -1.706899642944336,
+      "sft_loss": 1.4689277410507202,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 7.626931519624881,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": -0.06780585646629333,
+      "logits/rejected": 0.09956692159175873,
+      "logps/chosen": -1.4898687601089478,
+      "logps/rejected": -1.7180099487304688,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4898687601089478,
+      "rewards/margins": 0.22814103960990906,
+      "rewards/rejected": -1.7180099487304688,
+      "sft_loss": 1.537276268005371,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 5.433214321002391,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": -0.02779715694487095,
+      "logits/rejected": 0.11424566805362701,
+      "logps/chosen": -1.3562198877334595,
+      "logps/rejected": -1.6049524545669556,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3562198877334595,
+      "rewards/margins": 0.2487325221300125,
+      "rewards/rejected": -1.6049524545669556,
+      "sft_loss": 1.3964496850967407,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 8.29469757854584,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": -0.17457318305969238,
+      "logits/rejected": -0.01711769960820675,
+      "logps/chosen": -1.4438587427139282,
+      "logps/rejected": -1.6200523376464844,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4438587427139282,
+      "rewards/margins": 0.17619381844997406,
+      "rewards/rejected": -1.6200523376464844,
+      "sft_loss": 1.4548752307891846,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 5.888598056038247,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": -0.08861254900693893,
+      "logits/rejected": -0.07075365632772446,
+      "logps/chosen": -1.422069787979126,
+      "logps/rejected": -1.7725293636322021,
+      "loss": 0.5059,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.422069787979126,
+      "rewards/margins": 0.3504595160484314,
+      "rewards/rejected": -1.7725293636322021,
+      "sft_loss": 1.4434196949005127,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 5.2887307915100505,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": -0.09308328479528427,
+      "logits/rejected": 0.16950467228889465,
+      "logps/chosen": -1.3764543533325195,
+      "logps/rejected": -1.7472703456878662,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3764543533325195,
+      "rewards/margins": 0.3708159327507019,
+      "rewards/rejected": -1.7472703456878662,
+      "sft_loss": 1.458449125289917,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 5.01071388324762,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": 0.007398007903248072,
+      "logits/rejected": 0.06001367047429085,
+      "logps/chosen": -1.4065968990325928,
+      "logps/rejected": -1.6868667602539062,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4065968990325928,
+      "rewards/margins": 0.2802698314189911,
+      "rewards/rejected": -1.6868667602539062,
+      "sft_loss": 1.4229308366775513,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 3.931757603333999,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": 0.00926266424357891,
+      "logits/rejected": 0.09389156103134155,
+      "logps/chosen": -1.3826566934585571,
+      "logps/rejected": -1.6744199991226196,
+      "loss": 0.5238,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3826566934585571,
+      "rewards/margins": 0.29176321625709534,
+      "rewards/rejected": -1.6744199991226196,
+      "sft_loss": 1.4121711254119873,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 6.229548861295008,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": -0.0754542127251625,
+      "logits/rejected": 0.10710541903972626,
+      "logps/chosen": -1.357774019241333,
+      "logps/rejected": -1.7196426391601562,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.357774019241333,
+      "rewards/margins": 0.36186856031417847,
+      "rewards/rejected": -1.7196426391601562,
+      "sft_loss": 1.3753013610839844,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 4.592870510848051,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": -0.18370278179645538,
+      "logits/rejected": 0.020677369087934494,
+      "logps/chosen": -1.3615787029266357,
+      "logps/rejected": -1.6913648843765259,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3615787029266357,
+      "rewards/margins": 0.3297862112522125,
+      "rewards/rejected": -1.6913648843765259,
+      "sft_loss": 1.3998141288757324,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 7.837754456611585,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": -0.03532402962446213,
+      "logits/rejected": 0.08233954012393951,
+      "logps/chosen": -1.4457212686538696,
+      "logps/rejected": -1.7230970859527588,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4457212686538696,
+      "rewards/margins": 0.27737584710121155,
+      "rewards/rejected": -1.7230970859527588,
+      "sft_loss": 1.4261298179626465,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 6.3131827199709045,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": -0.04977645352482796,
+      "logits/rejected": 0.0691833645105362,
+      "logps/chosen": -1.3893170356750488,
+      "logps/rejected": -1.633607268333435,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3893170356750488,
+      "rewards/margins": 0.2442903220653534,
+      "rewards/rejected": -1.633607268333435,
+      "sft_loss": 1.399949550628662,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 7.695878675767656,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": -0.033573225140571594,
+      "logits/rejected": 0.046844761818647385,
+      "logps/chosen": -1.4621721506118774,
+      "logps/rejected": -1.711739182472229,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4621721506118774,
+      "rewards/margins": 0.24956703186035156,
+      "rewards/rejected": -1.711739182472229,
+      "sft_loss": 1.496930480003357,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 6.985545650344274,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": -0.01977308839559555,
+      "logits/rejected": 0.021173939108848572,
+      "logps/chosen": -1.3878123760223389,
+      "logps/rejected": -1.6171363592147827,
+      "loss": 0.5358,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.3878123760223389,
+      "rewards/margins": 0.22932401299476624,
+      "rewards/rejected": -1.6171363592147827,
+      "sft_loss": 1.4066617488861084,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 4.366435796291409,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": 0.02655363641679287,
+      "logits/rejected": 0.2236282378435135,
+      "logps/chosen": -1.382165551185608,
+      "logps/rejected": -1.6554224491119385,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.382165551185608,
+      "rewards/margins": 0.2732568681240082,
+      "rewards/rejected": -1.6554224491119385,
+      "sft_loss": 1.42926025390625,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 7.81159246638897,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": 0.03293774276971817,
+      "logits/rejected": 0.09691715985536575,
+      "logps/chosen": -1.4025884866714478,
+      "logps/rejected": -1.5003225803375244,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4025884866714478,
+      "rewards/margins": 0.09773415327072144,
+      "rewards/rejected": -1.5003225803375244,
+      "sft_loss": 1.440049409866333,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 5.268029857993076,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": 0.0003868401108775288,
+      "logits/rejected": 0.30421125888824463,
+      "logps/chosen": -1.4475411176681519,
+      "logps/rejected": -1.6507571935653687,
+      "loss": 0.558,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4475411176681519,
+      "rewards/margins": 0.20321616530418396,
+      "rewards/rejected": -1.6507571935653687,
+      "sft_loss": 1.478035569190979,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 5.476104551012313,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": -0.1295911818742752,
+      "logits/rejected": 0.025414815172553062,
+      "logps/chosen": -1.4887486696243286,
+      "logps/rejected": -1.822188138961792,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4887486696243286,
+      "rewards/margins": 0.3334396481513977,
+      "rewards/rejected": -1.822188138961792,
+      "sft_loss": 1.5442901849746704,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 4.149863738674073,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": -0.029890060424804688,
+      "logits/rejected": 0.10929396003484726,
+      "logps/chosen": -1.413294792175293,
+      "logps/rejected": -1.7026281356811523,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.413294792175293,
+      "rewards/margins": 0.2893335521221161,
+      "rewards/rejected": -1.7026281356811523,
+      "sft_loss": 1.4526488780975342,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 21.2802024194351,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": -0.018464690074324608,
+      "logits/rejected": 0.10323115438222885,
+      "logps/chosen": -1.5000975131988525,
+      "logps/rejected": -1.958627700805664,
+      "loss": 0.4919,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5000975131988525,
+      "rewards/margins": 0.45853009819984436,
+      "rewards/rejected": -1.958627700805664,
+      "sft_loss": 1.5218764543533325,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 3.655166951636573,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": -0.04375447332859039,
+      "logits/rejected": 0.11701725423336029,
+      "logps/chosen": -1.4266334772109985,
+      "logps/rejected": -1.8739063739776611,
+      "loss": 0.4893,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4266334772109985,
+      "rewards/margins": 0.4472728669643402,
+      "rewards/rejected": -1.8739063739776611,
+      "sft_loss": 1.4907891750335693,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 5.149895071058057,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": -0.08814994990825653,
+      "logits/rejected": 0.051434438675642014,
+      "logps/chosen": -1.497077465057373,
+      "logps/rejected": -1.6954923868179321,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.497077465057373,
+      "rewards/margins": 0.19841498136520386,
+      "rewards/rejected": -1.6954923868179321,
+      "sft_loss": 1.4636865854263306,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 8.096091593787765,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": -0.06694310158491135,
+      "logits/rejected": 0.07419678568840027,
+      "logps/chosen": -1.371813178062439,
+      "logps/rejected": -1.674665093421936,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.371813178062439,
+      "rewards/margins": 0.302852064371109,
+      "rewards/rejected": -1.674665093421936,
+      "sft_loss": 1.4243870973587036,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 7.542392017959355,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": 0.03423604369163513,
+      "logits/rejected": 0.15960803627967834,
+      "logps/chosen": -1.4509718418121338,
+      "logps/rejected": -1.7408100366592407,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4509718418121338,
+      "rewards/margins": 0.28983819484710693,
+      "rewards/rejected": -1.7408100366592407,
+      "sft_loss": 1.4992636442184448,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 4.278440356940747,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": 0.060794007033109665,
+      "logits/rejected": 0.15525543689727783,
+      "logps/chosen": -1.4032586812973022,
+      "logps/rejected": -1.6212393045425415,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4032586812973022,
+      "rewards/margins": 0.21798057854175568,
+      "rewards/rejected": -1.6212393045425415,
+      "sft_loss": 1.426965594291687,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 6.165930757755512,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": -0.13838526606559753,
+      "logits/rejected": 0.014215810224413872,
+      "logps/chosen": -1.528344988822937,
+      "logps/rejected": -1.7559783458709717,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.528344988822937,
+      "rewards/margins": 0.22763332724571228,
+      "rewards/rejected": -1.7559783458709717,
+      "sft_loss": 1.5378443002700806,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 4.358325387270058,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": -0.03495486080646515,
+      "logits/rejected": 0.15800495445728302,
+      "logps/chosen": -1.3264365196228027,
+      "logps/rejected": -1.7632360458374023,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3264365196228027,
+      "rewards/margins": 0.43679970502853394,
+      "rewards/rejected": -1.7632360458374023,
+      "sft_loss": 1.4041543006896973,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 8.185094426743838,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": -0.06456667929887772,
+      "logits/rejected": 0.1312786191701889,
+      "logps/chosen": -1.472119688987732,
+      "logps/rejected": -1.795741319656372,
+      "loss": 0.5151,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.472119688987732,
+      "rewards/margins": 0.3236214816570282,
+      "rewards/rejected": -1.795741319656372,
+      "sft_loss": 1.535501480102539,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 7.66034027661226,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": 0.0350164994597435,
+      "logits/rejected": 0.19542302191257477,
+      "logps/chosen": -1.4765806198120117,
+      "logps/rejected": -1.6078450679779053,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4765806198120117,
+      "rewards/margins": 0.1312645971775055,
+      "rewards/rejected": -1.6078450679779053,
+      "sft_loss": 1.4553964138031006,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 5.423041434028237,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": 0.04505506902933121,
+      "logits/rejected": 0.1933830976486206,
+      "logps/chosen": -1.3929471969604492,
+      "logps/rejected": -1.8875548839569092,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3929471969604492,
+      "rewards/margins": 0.4946078360080719,
+      "rewards/rejected": -1.8875548839569092,
+      "sft_loss": 1.4555342197418213,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 5.903348939683053,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": 0.05219608545303345,
+      "logits/rejected": 0.15234455466270447,
+      "logps/chosen": -1.4215357303619385,
+      "logps/rejected": -1.8510538339614868,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4215357303619385,
+      "rewards/margins": 0.4295181334018707,
+      "rewards/rejected": -1.8510538339614868,
+      "sft_loss": 1.4525610208511353,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 8.799420024776662,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": -0.045768219977617264,
+      "logits/rejected": 0.10385224968194962,
+      "logps/chosen": -1.5010254383087158,
+      "logps/rejected": -1.7813167572021484,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5010254383087158,
+      "rewards/margins": 0.2802914083003998,
+      "rewards/rejected": -1.7813167572021484,
+      "sft_loss": 1.4974098205566406,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 4.098418469896391,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": 0.11667392402887344,
+      "logits/rejected": 0.1806088089942932,
+      "logps/chosen": -1.3921082019805908,
+      "logps/rejected": -1.8272292613983154,
+      "loss": 0.4879,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3921082019805908,
+      "rewards/margins": 0.4351211488246918,
+      "rewards/rejected": -1.8272292613983154,
+      "sft_loss": 1.4282985925674438,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 4.37448697414661,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": 0.0001061245784512721,
+      "logits/rejected": 0.1211082935333252,
+      "logps/chosen": -1.4377018213272095,
+      "logps/rejected": -1.6614086627960205,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4377018213272095,
+      "rewards/margins": 0.2237069308757782,
+      "rewards/rejected": -1.6614086627960205,
+      "sft_loss": 1.5097572803497314,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 7.801656820601944,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": -0.0659121721982956,
+      "logits/rejected": 0.16661436855793,
+      "logps/chosen": -1.4803264141082764,
+      "logps/rejected": -1.7045660018920898,
+      "loss": 0.5617,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4803264141082764,
+      "rewards/margins": 0.22423963248729706,
+      "rewards/rejected": -1.7045660018920898,
+      "sft_loss": 1.5604345798492432,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 9.88040393877702,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": 0.060816846787929535,
+      "logits/rejected": 0.16335485875606537,
+      "logps/chosen": -1.4167139530181885,
+      "logps/rejected": -1.654566764831543,
+      "loss": 0.5477,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4167139530181885,
+      "rewards/margins": 0.23785285651683807,
+      "rewards/rejected": -1.654566764831543,
+      "sft_loss": 1.4502527713775635,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 6.799381513616858,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": -0.011297956109046936,
+      "logits/rejected": 0.16043448448181152,
+      "logps/chosen": -1.3892749547958374,
+      "logps/rejected": -1.7083572149276733,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3892749547958374,
+      "rewards/margins": 0.31908220052719116,
+      "rewards/rejected": -1.7083572149276733,
+      "sft_loss": 1.4365092515945435,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 4.425286877544062,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": 0.01945159211754799,
+      "logits/rejected": 0.20056121051311493,
+      "logps/chosen": -1.3618758916854858,
+      "logps/rejected": -1.7152411937713623,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3618758916854858,
+      "rewards/margins": 0.35336512327194214,
+      "rewards/rejected": -1.7152411937713623,
+      "sft_loss": 1.3751140832901,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 7.25832869318277,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": 0.060930054634809494,
+      "logits/rejected": 0.18895120918750763,
+      "logps/chosen": -1.4787466526031494,
+      "logps/rejected": -1.742435097694397,
+      "loss": 0.548,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4787466526031494,
+      "rewards/margins": 0.26368841528892517,
+      "rewards/rejected": -1.742435097694397,
+      "sft_loss": 1.5360684394836426,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 4.933151756958093,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": -0.023029694333672523,
+      "logits/rejected": 0.009718982502818108,
+      "logps/chosen": -1.4260733127593994,
+      "logps/rejected": -1.7250900268554688,
+      "loss": 0.5289,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4260733127593994,
+      "rewards/margins": 0.29901689291000366,
+      "rewards/rejected": -1.7250900268554688,
+      "sft_loss": 1.4916422367095947,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 6.560511701032652,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": -0.0503508560359478,
+      "logits/rejected": 0.13822703063488007,
+      "logps/chosen": -1.4035285711288452,
+      "logps/rejected": -1.7277904748916626,
+      "loss": 0.5121,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4035285711288452,
+      "rewards/margins": 0.3242620825767517,
+      "rewards/rejected": -1.7277904748916626,
+      "sft_loss": 1.4790805578231812,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 4.958156966185051,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": 0.051444463431835175,
+      "logits/rejected": 0.17066633701324463,
+      "logps/chosen": -1.3592405319213867,
+      "logps/rejected": -1.7749545574188232,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3592405319213867,
+      "rewards/margins": 0.4157140254974365,
+      "rewards/rejected": -1.7749545574188232,
+      "sft_loss": 1.4301153421401978,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 5.009038213095197,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": 0.07895182073116302,
+      "logits/rejected": 0.2562471032142639,
+      "logps/chosen": -1.4837896823883057,
+      "logps/rejected": -1.892653226852417,
+      "loss": 0.5159,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4837896823883057,
+      "rewards/margins": 0.4088636338710785,
+      "rewards/rejected": -1.892653226852417,
+      "sft_loss": 1.4823133945465088,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 6.097392272653846,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": 0.00459608668461442,
+      "logits/rejected": 0.13557849824428558,
+      "logps/chosen": -1.4189090728759766,
+      "logps/rejected": -1.684511423110962,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4189090728759766,
+      "rewards/margins": 0.2656022608280182,
+      "rewards/rejected": -1.684511423110962,
+      "sft_loss": 1.4373775720596313,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 9.718433536349252,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": 0.003417456056922674,
+      "logits/rejected": 0.1183265894651413,
+      "logps/chosen": -1.586362600326538,
+      "logps/rejected": -1.7566999197006226,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.586362600326538,
+      "rewards/margins": 0.1703372746706009,
+      "rewards/rejected": -1.7566999197006226,
+      "sft_loss": 1.6519889831542969,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 10.362820259678747,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": -0.0650627538561821,
+      "logits/rejected": 0.04227939993143082,
+      "logps/chosen": -1.4759619235992432,
+      "logps/rejected": -1.8760197162628174,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4759619235992432,
+      "rewards/margins": 0.40005773305892944,
+      "rewards/rejected": -1.8760197162628174,
+      "sft_loss": 1.5462987422943115,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 7.091050987105003,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": 0.05563043802976608,
+      "logits/rejected": 0.13614422082901,
+      "logps/chosen": -1.4341747760772705,
+      "logps/rejected": -1.8504860401153564,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4341747760772705,
+      "rewards/margins": 0.416311115026474,
+      "rewards/rejected": -1.8504860401153564,
+      "sft_loss": 1.4413312673568726,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 3.8930626725283717,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": -0.03380414843559265,
+      "logits/rejected": 0.27143990993499756,
+      "logps/chosen": -1.4646518230438232,
+      "logps/rejected": -1.8340686559677124,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4646518230438232,
+      "rewards/margins": 0.36941686272621155,
+      "rewards/rejected": -1.8340686559677124,
+      "sft_loss": 1.4922515153884888,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 5.530315878153944,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": 0.06472794711589813,
+      "logits/rejected": 0.1404540240764618,
+      "logps/chosen": -1.4131653308868408,
+      "logps/rejected": -1.723859429359436,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4131653308868408,
+      "rewards/margins": 0.3106943964958191,
+      "rewards/rejected": -1.723859429359436,
+      "sft_loss": 1.4428884983062744,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 6.649544973120154,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": 0.02315221168100834,
+      "logits/rejected": 0.11982355266809464,
+      "logps/chosen": -1.4375369548797607,
+      "logps/rejected": -1.7398595809936523,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4375369548797607,
+      "rewards/margins": 0.3023225963115692,
+      "rewards/rejected": -1.7398595809936523,
+      "sft_loss": 1.5597889423370361,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 4.798262164071743,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": -0.014184604398906231,
+      "logits/rejected": 0.18807581067085266,
+      "logps/chosen": -1.4288346767425537,
+      "logps/rejected": -1.6621601581573486,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4288346767425537,
+      "rewards/margins": 0.23332540690898895,
+      "rewards/rejected": -1.6621601581573486,
+      "sft_loss": 1.4449572563171387,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 7.721218526827049,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": -0.05206465721130371,
+      "logits/rejected": 0.06865311414003372,
+      "logps/chosen": -1.4678184986114502,
+      "logps/rejected": -1.7325515747070312,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4678184986114502,
+      "rewards/margins": 0.2647330164909363,
+      "rewards/rejected": -1.7325515747070312,
+      "sft_loss": 1.4634746313095093,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 7.566815023460756,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": -0.04291130229830742,
+      "logits/rejected": 0.14794987440109253,
+      "logps/chosen": -1.3930656909942627,
+      "logps/rejected": -1.6991512775421143,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3930656909942627,
+      "rewards/margins": 0.3060855269432068,
+      "rewards/rejected": -1.6991512775421143,
+      "sft_loss": 1.4516359567642212,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 8.855577325246788,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": 0.03489986062049866,
+      "logits/rejected": 0.21353301405906677,
+      "logps/chosen": -1.4300438165664673,
+      "logps/rejected": -1.8971660137176514,
+      "loss": 0.4705,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4300438165664673,
+      "rewards/margins": 0.46712246537208557,
+      "rewards/rejected": -1.8971660137176514,
+      "sft_loss": 1.4638926982879639,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 5.968612534315002,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": 0.04957198724150658,
+      "logits/rejected": 0.23397859930992126,
+      "logps/chosen": -1.448665738105774,
+      "logps/rejected": -1.7475725412368774,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.448665738105774,
+      "rewards/margins": 0.29890674352645874,
+      "rewards/rejected": -1.7475725412368774,
+      "sft_loss": 1.497968077659607,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 5.774737608548478,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": -0.10732851177453995,
+      "logits/rejected": 0.19018636643886566,
+      "logps/chosen": -1.4928748607635498,
+      "logps/rejected": -1.8812583684921265,
+      "loss": 0.5216,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4928748607635498,
+      "rewards/margins": 0.3883834481239319,
+      "rewards/rejected": -1.8812583684921265,
+      "sft_loss": 1.5361144542694092,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 5.013952174103058,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": -0.0828329548239708,
+      "logits/rejected": 0.19987811148166656,
+      "logps/chosen": -1.5106512308120728,
+      "logps/rejected": -1.9506137371063232,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5106512308120728,
+      "rewards/margins": 0.4399626851081848,
+      "rewards/rejected": -1.9506137371063232,
+      "sft_loss": 1.4890204668045044,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 7.321461068617166,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": 0.04645683243870735,
+      "logits/rejected": 0.1263210028409958,
+      "logps/chosen": -1.563812017440796,
+      "logps/rejected": -1.8573408126831055,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.563812017440796,
+      "rewards/margins": 0.2935287654399872,
+      "rewards/rejected": -1.8573408126831055,
+      "sft_loss": 1.6140215396881104,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 4.6835886280890735,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": 0.018405133858323097,
+      "logits/rejected": 0.11775252968072891,
+      "logps/chosen": -1.52276611328125,
+      "logps/rejected": -1.820581078529358,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.52276611328125,
+      "rewards/margins": 0.29781508445739746,
+      "rewards/rejected": -1.820581078529358,
+      "sft_loss": 1.5765827894210815,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 13.799267959660574,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": 0.054694343358278275,
+      "logits/rejected": 0.08119392395019531,
+      "logps/chosen": -1.425003170967102,
+      "logps/rejected": -1.666555643081665,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.425003170967102,
+      "rewards/margins": 0.24155232310295105,
+      "rewards/rejected": -1.666555643081665,
+      "sft_loss": 1.4962987899780273,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 5.962658386421319,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": 0.006678787060081959,
+      "logits/rejected": 0.12651744484901428,
+      "logps/chosen": -1.430008053779602,
+      "logps/rejected": -1.665032148361206,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.430008053779602,
+      "rewards/margins": 0.23502404987812042,
+      "rewards/rejected": -1.665032148361206,
+      "sft_loss": 1.466813325881958,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 7.66418865064177,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": 0.0546504482626915,
+      "logits/rejected": 0.3250477910041809,
+      "logps/chosen": -1.4601762294769287,
+      "logps/rejected": -1.8048124313354492,
+      "loss": 0.5084,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4601762294769287,
+      "rewards/margins": 0.34463605284690857,
+      "rewards/rejected": -1.8048124313354492,
+      "sft_loss": 1.496201992034912,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 4.515357989887072,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": 0.0008736684685572982,
+      "logits/rejected": 0.1726742684841156,
+      "logps/chosen": -1.3656678199768066,
+      "logps/rejected": -1.7986243963241577,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3656678199768066,
+      "rewards/margins": 0.4329564571380615,
+      "rewards/rejected": -1.7986243963241577,
+      "sft_loss": 1.3878506422042847,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 10.693019607988258,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": -0.07369935512542725,
+      "logits/rejected": 0.027142727747559547,
+      "logps/chosen": -1.3922083377838135,
+      "logps/rejected": -1.6310374736785889,
+      "loss": 0.5281,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3922083377838135,
+      "rewards/margins": 0.23882922530174255,
+      "rewards/rejected": -1.6310374736785889,
+      "sft_loss": 1.4906740188598633,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 8.812585236392314,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": 0.014504333958029747,
+      "logits/rejected": 0.1055087074637413,
+      "logps/chosen": -1.5411064624786377,
+      "logps/rejected": -1.707898497581482,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.5411064624786377,
+      "rewards/margins": 0.16679205000400543,
+      "rewards/rejected": -1.707898497581482,
+      "sft_loss": 1.5593599081039429,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 3.5492870174689735,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": -0.10841546207666397,
+      "logits/rejected": 0.10659325122833252,
+      "logps/chosen": -1.4652018547058105,
+      "logps/rejected": -1.883522391319275,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4652018547058105,
+      "rewards/margins": 0.41832059621810913,
+      "rewards/rejected": -1.883522391319275,
+      "sft_loss": 1.5147536993026733,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": 0.27520132064819336,
+      "eval_logits/rejected": 0.37425851821899414,
+      "eval_logps/chosen": -1.4448524713516235,
+      "eval_logps/rejected": -1.7957645654678345,
+      "eval_loss": 0.5101240873336792,
+      "eval_rewards/accuracies": 0.6261127591133118,
+      "eval_rewards/chosen": -1.4448524713516235,
+      "eval_rewards/margins": 0.3509122133255005,
+      "eval_rewards/rejected": -1.7957645654678345,
+      "eval_runtime": 43.2512,
+      "eval_samples_per_second": 31.097,
+      "eval_sft_loss": 1.497976541519165,
+      "eval_steps_per_second": 7.792,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 6.1013220165784485,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": -0.09389957785606384,
+      "logits/rejected": 0.15035173296928406,
+      "logps/chosen": -1.4221055507659912,
+      "logps/rejected": -1.7961909770965576,
+      "loss": 0.5,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4221055507659912,
+      "rewards/margins": 0.3740856349468231,
+      "rewards/rejected": -1.7961909770965576,
+      "sft_loss": 1.487693428993225,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 4.059476077333794,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": -0.09287907928228378,
+      "logits/rejected": 0.10007772594690323,
+      "logps/chosen": -1.4006273746490479,
+      "logps/rejected": -1.8340444564819336,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4006273746490479,
+      "rewards/margins": 0.43341708183288574,
+      "rewards/rejected": -1.8340444564819336,
+      "sft_loss": 1.4587966203689575,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 7.136162921385605,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": -0.05847252532839775,
+      "logits/rejected": 0.06235013157129288,
+      "logps/chosen": -1.4863847494125366,
+      "logps/rejected": -1.8875106573104858,
+      "loss": 0.5281,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4863847494125366,
+      "rewards/margins": 0.401125967502594,
+      "rewards/rejected": -1.8875106573104858,
+      "sft_loss": 1.5500595569610596,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 5.857959777310137,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": -0.028884366154670715,
+      "logits/rejected": 0.1254526674747467,
+      "logps/chosen": -1.370598316192627,
+      "logps/rejected": -1.7824159860610962,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.370598316192627,
+      "rewards/margins": 0.4118177890777588,
+      "rewards/rejected": -1.7824159860610962,
+      "sft_loss": 1.4120441675186157,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 5.859143629683388,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": 0.06796115636825562,
+      "logits/rejected": 0.11744487285614014,
+      "logps/chosen": -1.5567047595977783,
+      "logps/rejected": -1.7756681442260742,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5567047595977783,
+      "rewards/margins": 0.21896354854106903,
+      "rewards/rejected": -1.7756681442260742,
+      "sft_loss": 1.5273792743682861,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 5.929078630089166,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": -0.02454587072134018,
+      "logits/rejected": 0.1381051242351532,
+      "logps/chosen": -1.4669086933135986,
+      "logps/rejected": -1.8011709451675415,
+      "loss": 0.528,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4669086933135986,
+      "rewards/margins": 0.33426228165626526,
+      "rewards/rejected": -1.8011709451675415,
+      "sft_loss": 1.487455129623413,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 5.423182196882485,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": -0.1283598393201828,
+      "logits/rejected": 0.0018849506741389632,
+      "logps/chosen": -1.3589659929275513,
+      "logps/rejected": -1.7768265008926392,
+      "loss": 0.489,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3589659929275513,
+      "rewards/margins": 0.4178605079650879,
+      "rewards/rejected": -1.7768265008926392,
+      "sft_loss": 1.4241197109222412,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 5.874207795306028,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": -0.09055529534816742,
+      "logits/rejected": 0.0466456413269043,
+      "logps/chosen": -1.4008266925811768,
+      "logps/rejected": -1.8529808521270752,
+      "loss": 0.49,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4008266925811768,
+      "rewards/margins": 0.4521542489528656,
+      "rewards/rejected": -1.8529808521270752,
+      "sft_loss": 1.4470208883285522,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 5.7827022662756065,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": -0.008395682089030743,
+      "logits/rejected": 0.056283582001924515,
+      "logps/chosen": -1.3998442888259888,
+      "logps/rejected": -1.6840111017227173,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3998442888259888,
+      "rewards/margins": 0.2841669023036957,
+      "rewards/rejected": -1.6840111017227173,
+      "sft_loss": 1.4410978555679321,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 7.954396025512038,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": 0.049496881663799286,
+      "logits/rejected": 0.12194067239761353,
+      "logps/chosen": -1.410659670829773,
+      "logps/rejected": -1.7615478038787842,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.410659670829773,
+      "rewards/margins": 0.35088807344436646,
+      "rewards/rejected": -1.7615478038787842,
+      "sft_loss": 1.4320547580718994,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 5.994738412694896,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": -0.06979848444461823,
+      "logits/rejected": 0.055761612951755524,
+      "logps/chosen": -1.4576666355133057,
+      "logps/rejected": -1.745171308517456,
+      "loss": 0.5301,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4576666355133057,
+      "rewards/margins": 0.2875046133995056,
+      "rewards/rejected": -1.745171308517456,
+      "sft_loss": 1.4637020826339722,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 5.199844719026512,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": 0.007501667831093073,
+      "logits/rejected": 0.036503929644823074,
+      "logps/chosen": -1.3404067754745483,
+      "logps/rejected": -1.6517269611358643,
+      "loss": 0.5083,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3404067754745483,
+      "rewards/margins": 0.3113202154636383,
+      "rewards/rejected": -1.6517269611358643,
+      "sft_loss": 1.310537338256836,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 3.984860240033483,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": -0.025994906201958656,
+      "logits/rejected": 0.048259999603033066,
+      "logps/chosen": -1.4862369298934937,
+      "logps/rejected": -1.7005221843719482,
+      "loss": 0.5652,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4862369298934937,
+      "rewards/margins": 0.21428528428077698,
+      "rewards/rejected": -1.7005221843719482,
+      "sft_loss": 1.4881370067596436,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 5.232632139565415,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": 0.02264488860964775,
+      "logits/rejected": 0.20778794586658478,
+      "logps/chosen": -1.504136323928833,
+      "logps/rejected": -1.7099090814590454,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.504136323928833,
+      "rewards/margins": 0.20577266812324524,
+      "rewards/rejected": -1.7099090814590454,
+      "sft_loss": 1.464942216873169,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 5.495620916985516,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": 0.028523176908493042,
+      "logits/rejected": 0.25938358902931213,
+      "logps/chosen": -1.4731388092041016,
+      "logps/rejected": -1.8747011423110962,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4731388092041016,
+      "rewards/margins": 0.40156230330467224,
+      "rewards/rejected": -1.8747011423110962,
+      "sft_loss": 1.527525544166565,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 6.408484159485986,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": -0.07703231275081635,
+      "logits/rejected": 0.050923220813274384,
+      "logps/chosen": -1.4904206991195679,
+      "logps/rejected": -1.7379696369171143,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4904206991195679,
+      "rewards/margins": 0.24754881858825684,
+      "rewards/rejected": -1.7379696369171143,
+      "sft_loss": 1.5126824378967285,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 6.594947547527362,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": -0.0388483852148056,
+      "logits/rejected": 0.2138877660036087,
+      "logps/chosen": -1.4265109300613403,
+      "logps/rejected": -1.7498865127563477,
+      "loss": 0.5476,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4265109300613403,
+      "rewards/margins": 0.32337552309036255,
+      "rewards/rejected": -1.7498865127563477,
+      "sft_loss": 1.4754232168197632,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 5.115040645172101,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": -0.035510290414094925,
+      "logits/rejected": 0.17635096609592438,
+      "logps/chosen": -1.3613451719284058,
+      "logps/rejected": -1.716199517250061,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3613451719284058,
+      "rewards/margins": 0.3548543155193329,
+      "rewards/rejected": -1.716199517250061,
+      "sft_loss": 1.4790536165237427,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 8.074996039783558,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": -0.16678813099861145,
+      "logits/rejected": 0.001837402582168579,
+      "logps/chosen": -1.4480372667312622,
+      "logps/rejected": -1.7357591390609741,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4480372667312622,
+      "rewards/margins": 0.2877218723297119,
+      "rewards/rejected": -1.7357591390609741,
+      "sft_loss": 1.5501583814620972,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 6.995634014615492,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": 0.01545445155352354,
+      "logits/rejected": -0.03494206815958023,
+      "logps/chosen": -1.494322419166565,
+      "logps/rejected": -1.631588339805603,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.494322419166565,
+      "rewards/margins": 0.13726598024368286,
+      "rewards/rejected": -1.631588339805603,
+      "sft_loss": 1.5300920009613037,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 9.560309320263531,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": -0.010472054593265057,
+      "logits/rejected": 0.15402260422706604,
+      "logps/chosen": -1.4279364347457886,
+      "logps/rejected": -1.6980457305908203,
+      "loss": 0.5178,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4279364347457886,
+      "rewards/margins": 0.2701093554496765,
+      "rewards/rejected": -1.6980457305908203,
+      "sft_loss": 1.4880188703536987,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 5.428743027890913,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": 0.0033090352080762386,
+      "logits/rejected": 0.10284850746393204,
+      "logps/chosen": -1.4249826669692993,
+      "logps/rejected": -1.7031883001327515,
+      "loss": 0.5176,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4249826669692993,
+      "rewards/margins": 0.27820563316345215,
+      "rewards/rejected": -1.7031883001327515,
+      "sft_loss": 1.4983646869659424,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 5.7398379298109345,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": -0.02407381683588028,
+      "logits/rejected": 0.2022133320569992,
+      "logps/chosen": -1.5204074382781982,
+      "logps/rejected": -1.7571582794189453,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5204074382781982,
+      "rewards/margins": 0.23675091564655304,
+      "rewards/rejected": -1.7571582794189453,
+      "sft_loss": 1.5273138284683228,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 5.898544013989893,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": -0.024211686104536057,
+      "logits/rejected": 0.05473152548074722,
+      "logps/chosen": -1.3195667266845703,
+      "logps/rejected": -1.7765014171600342,
+      "loss": 0.47,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3195667266845703,
+      "rewards/margins": 0.45693454146385193,
+      "rewards/rejected": -1.7765014171600342,
+      "sft_loss": 1.383522868156433,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 4.328819037966786,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": -0.010368740186095238,
+      "logits/rejected": 0.11343619972467422,
+      "logps/chosen": -1.508044958114624,
+      "logps/rejected": -1.948987364768982,
+      "loss": 0.5062,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.508044958114624,
+      "rewards/margins": 0.4409424364566803,
+      "rewards/rejected": -1.948987364768982,
+      "sft_loss": 1.5886409282684326,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 6.320156944153655,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": 0.07320670038461685,
+      "logits/rejected": 0.13631558418273926,
+      "logps/chosen": -1.4563207626342773,
+      "logps/rejected": -1.6941626071929932,
+      "loss": 0.5495,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4563207626342773,
+      "rewards/margins": 0.2378418892621994,
+      "rewards/rejected": -1.6941626071929932,
+      "sft_loss": 1.4958994388580322,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 8.404853592021109,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": -0.11669330298900604,
+      "logits/rejected": 0.07316017150878906,
+      "logps/chosen": -1.4231197834014893,
+      "logps/rejected": -2.047053813934326,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4231197834014893,
+      "rewards/margins": 0.6239344477653503,
+      "rewards/rejected": -2.047053813934326,
+      "sft_loss": 1.414732575416565,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 8.028394240012739,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": -0.0647435188293457,
+      "logits/rejected": 0.10035815089941025,
+      "logps/chosen": -1.4947988986968994,
+      "logps/rejected": -1.8533751964569092,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4947988986968994,
+      "rewards/margins": 0.35857629776000977,
+      "rewards/rejected": -1.8533751964569092,
+      "sft_loss": 1.4600107669830322,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 4.238728475751365,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": 0.022504812106490135,
+      "logits/rejected": 0.06147592142224312,
+      "logps/chosen": -1.4899694919586182,
+      "logps/rejected": -1.8011322021484375,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4899694919586182,
+      "rewards/margins": 0.3111625611782074,
+      "rewards/rejected": -1.8011322021484375,
+      "sft_loss": 1.5551286935806274,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 11.81033325638871,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": -0.05830759555101395,
+      "logits/rejected": 0.0124333705753088,
+      "logps/chosen": -1.5286645889282227,
+      "logps/rejected": -1.8843910694122314,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5286645889282227,
+      "rewards/margins": 0.3557264506816864,
+      "rewards/rejected": -1.8843910694122314,
+      "sft_loss": 1.543145775794983,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 5.7829093648908545,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": -0.05018848180770874,
+      "logits/rejected": 0.06818221509456635,
+      "logps/chosen": -1.4554150104522705,
+      "logps/rejected": -1.7889883518218994,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4554150104522705,
+      "rewards/margins": 0.33357328176498413,
+      "rewards/rejected": -1.7889883518218994,
+      "sft_loss": 1.4961915016174316,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 7.000753850441391,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": -0.031203657388687134,
+      "logits/rejected": 0.10760529339313507,
+      "logps/chosen": -1.415029764175415,
+      "logps/rejected": -1.769468903541565,
+      "loss": 0.5063,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.415029764175415,
+      "rewards/margins": 0.3544391989707947,
+      "rewards/rejected": -1.769468903541565,
+      "sft_loss": 1.5223928689956665,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 6.464241093793835,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": 0.033028002828359604,
+      "logits/rejected": 0.058578938245773315,
+      "logps/chosen": -1.4606726169586182,
+      "logps/rejected": -1.8109041452407837,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4606726169586182,
+      "rewards/margins": 0.35023149847984314,
+      "rewards/rejected": -1.8109041452407837,
+      "sft_loss": 1.487366795539856,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 5.275683302776248,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": 0.02882230281829834,
+      "logits/rejected": 0.19418470561504364,
+      "logps/chosen": -1.491390585899353,
+      "logps/rejected": -1.716875433921814,
+      "loss": 0.5343,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.491390585899353,
+      "rewards/margins": 0.22548499703407288,
+      "rewards/rejected": -1.716875433921814,
+      "sft_loss": 1.5724704265594482,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 5.048787352166431,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": -0.05322617292404175,
+      "logits/rejected": 0.053495001047849655,
+      "logps/chosen": -1.4078679084777832,
+      "logps/rejected": -1.7306697368621826,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4078679084777832,
+      "rewards/margins": 0.32280176877975464,
+      "rewards/rejected": -1.7306697368621826,
+      "sft_loss": 1.4821839332580566,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 6.082908963768483,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": -0.05241629481315613,
+      "logits/rejected": 0.08349411189556122,
+      "logps/chosen": -1.3877642154693604,
+      "logps/rejected": -1.7418733835220337,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3877642154693604,
+      "rewards/margins": 0.35410913825035095,
+      "rewards/rejected": -1.7418733835220337,
+      "sft_loss": 1.3849250078201294,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 5.8152861408899605,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": 0.07076708227396011,
+      "logits/rejected": 0.1529882252216339,
+      "logps/chosen": -1.3420288562774658,
+      "logps/rejected": -1.5302860736846924,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3420288562774658,
+      "rewards/margins": 0.1882571130990982,
+      "rewards/rejected": -1.5302860736846924,
+      "sft_loss": 1.3648477792739868,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 5.7034590119810185,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": 0.06448648869991302,
+      "logits/rejected": 0.12127365171909332,
+      "logps/chosen": -1.4409104585647583,
+      "logps/rejected": -1.6786985397338867,
+      "loss": 0.5331,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4409104585647583,
+      "rewards/margins": 0.2377881556749344,
+      "rewards/rejected": -1.6786985397338867,
+      "sft_loss": 1.49290931224823,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 4.6440102903748,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": -0.07223442196846008,
+      "logits/rejected": 0.08344900608062744,
+      "logps/chosen": -1.3767060041427612,
+      "logps/rejected": -1.7289140224456787,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3767060041427612,
+      "rewards/margins": 0.35220807790756226,
+      "rewards/rejected": -1.7289140224456787,
+      "sft_loss": 1.452522873878479,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 4.507598778221961,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": -0.004403007682412863,
+      "logits/rejected": 0.08500498533248901,
+      "logps/chosen": -1.4097429513931274,
+      "logps/rejected": -1.6911542415618896,
+      "loss": 0.5172,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4097429513931274,
+      "rewards/margins": 0.28141140937805176,
+      "rewards/rejected": -1.6911542415618896,
+      "sft_loss": 1.5189019441604614,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 5.460169181811743,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": -0.13715745508670807,
+      "logits/rejected": 0.010128025896847248,
+      "logps/chosen": -1.3865242004394531,
+      "logps/rejected": -1.8770020008087158,
+      "loss": 0.4638,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3865242004394531,
+      "rewards/margins": 0.4904778003692627,
+      "rewards/rejected": -1.8770020008087158,
+      "sft_loss": 1.4646705389022827,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 5.003102751583516,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": 0.021493779495358467,
+      "logits/rejected": 0.18332651257514954,
+      "logps/chosen": -1.4104686975479126,
+      "logps/rejected": -1.7444658279418945,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4104686975479126,
+      "rewards/margins": 0.3339969515800476,
+      "rewards/rejected": -1.7444658279418945,
+      "sft_loss": 1.4638065099716187,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 5.316062872583758,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": -0.033848315477371216,
+      "logits/rejected": 0.12872003018856049,
+      "logps/chosen": -1.456559419631958,
+      "logps/rejected": -1.7650468349456787,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.456559419631958,
+      "rewards/margins": 0.3084874749183655,
+      "rewards/rejected": -1.7650468349456787,
+      "sft_loss": 1.5225870609283447,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 6.469002214831444,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": -0.008161703124642372,
+      "logits/rejected": 0.07683838903903961,
+      "logps/chosen": -1.4914897680282593,
+      "logps/rejected": -1.796868085861206,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4914897680282593,
+      "rewards/margins": 0.3053787052631378,
+      "rewards/rejected": -1.796868085861206,
+      "sft_loss": 1.4638153314590454,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 6.337595333137927,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": -0.08625678718090057,
+      "logits/rejected": 0.08819940686225891,
+      "logps/chosen": -1.4118739366531372,
+      "logps/rejected": -1.8817695379257202,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4118739366531372,
+      "rewards/margins": 0.46989554166793823,
+      "rewards/rejected": -1.8817695379257202,
+      "sft_loss": 1.483617901802063,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 4.306246185410908,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": 0.09946813434362411,
+      "logits/rejected": 0.1469993144273758,
+      "logps/chosen": -1.5145267248153687,
+      "logps/rejected": -1.7509231567382812,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5145267248153687,
+      "rewards/margins": 0.2363964021205902,
+      "rewards/rejected": -1.7509231567382812,
+      "sft_loss": 1.5372315645217896,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 6.10389102120731,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": -0.002360707614570856,
+      "logits/rejected": 0.07914100587368011,
+      "logps/chosen": -1.4556949138641357,
+      "logps/rejected": -1.6280122995376587,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4556949138641357,
+      "rewards/margins": 0.17231719195842743,
+      "rewards/rejected": -1.6280122995376587,
+      "sft_loss": 1.5284003019332886,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 6.074650721812385,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": 0.06208980083465576,
+      "logits/rejected": 0.19643422961235046,
+      "logps/chosen": -1.467151403427124,
+      "logps/rejected": -1.8871538639068604,
+      "loss": 0.4898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.467151403427124,
+      "rewards/margins": 0.4200025200843811,
+      "rewards/rejected": -1.8871538639068604,
+      "sft_loss": 1.5178897380828857,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 7.47352279179211,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": -0.009203584864735603,
+      "logits/rejected": 0.10609463602304459,
+      "logps/chosen": -1.5118629932403564,
+      "logps/rejected": -1.8410472869873047,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5118629932403564,
+      "rewards/margins": 0.3291842043399811,
+      "rewards/rejected": -1.8410472869873047,
+      "sft_loss": 1.5683891773223877,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 12.76772038228278,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": -0.1566140055656433,
+      "logits/rejected": -0.010694952681660652,
+      "logps/chosen": -1.558301568031311,
+      "logps/rejected": -1.7742935419082642,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.558301568031311,
+      "rewards/margins": 0.2159920185804367,
+      "rewards/rejected": -1.7742935419082642,
+      "sft_loss": 1.5489089488983154,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 10.974762656119584,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": -0.05199091508984566,
+      "logits/rejected": 0.08179700374603271,
+      "logps/chosen": -1.4593642950057983,
+      "logps/rejected": -1.8170993328094482,
+      "loss": 0.5136,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4593642950057983,
+      "rewards/margins": 0.3577350974082947,
+      "rewards/rejected": -1.8170993328094482,
+      "sft_loss": 1.519140362739563,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 8.61140192519221,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": -0.03101665899157524,
+      "logits/rejected": 0.06896068155765533,
+      "logps/chosen": -1.592306137084961,
+      "logps/rejected": -1.8628515005111694,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.592306137084961,
+      "rewards/margins": 0.27054545283317566,
+      "rewards/rejected": -1.8628515005111694,
+      "sft_loss": 1.5682618618011475,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 5.49984973042548,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": -0.06608717143535614,
+      "logits/rejected": 0.02036227658390999,
+      "logps/chosen": -1.3796789646148682,
+      "logps/rejected": -1.7269248962402344,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3796789646148682,
+      "rewards/margins": 0.3472460210323334,
+      "rewards/rejected": -1.7269248962402344,
+      "sft_loss": 1.4405910968780518,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 8.509372779398811,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": -0.0892179012298584,
+      "logits/rejected": 0.0769272893667221,
+      "logps/chosen": -1.5132403373718262,
+      "logps/rejected": -1.8297183513641357,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5132403373718262,
+      "rewards/margins": 0.31647807359695435,
+      "rewards/rejected": -1.8297183513641357,
+      "sft_loss": 1.5695701837539673,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 10.940379009851805,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": -0.13610494136810303,
+      "logits/rejected": 0.0053990124724805355,
+      "logps/chosen": -1.499241590499878,
+      "logps/rejected": -1.732712984085083,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.499241590499878,
+      "rewards/margins": 0.23347143828868866,
+      "rewards/rejected": -1.732712984085083,
+      "sft_loss": 1.4607391357421875,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 8.276979456989288,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": -0.018297823145985603,
+      "logits/rejected": 0.16213171184062958,
+      "logps/chosen": -1.4397025108337402,
+      "logps/rejected": -1.926867127418518,
+      "loss": 0.487,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4397025108337402,
+      "rewards/margins": 0.4871644973754883,
+      "rewards/rejected": -1.926867127418518,
+      "sft_loss": 1.5095751285552979,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 7.535703865238154,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": -0.07311379909515381,
+      "logits/rejected": 0.011505231261253357,
+      "logps/chosen": -1.3841418027877808,
+      "logps/rejected": -1.7198749780654907,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3841418027877808,
+      "rewards/margins": 0.33573317527770996,
+      "rewards/rejected": -1.7198749780654907,
+      "sft_loss": 1.3955576419830322,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 6.248961178342216,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": -0.07998314499855042,
+      "logits/rejected": 0.127044677734375,
+      "logps/chosen": -1.4475855827331543,
+      "logps/rejected": -1.784197211265564,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4475855827331543,
+      "rewards/margins": 0.33661171793937683,
+      "rewards/rejected": -1.784197211265564,
+      "sft_loss": 1.5629757642745972,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 5.787207779107956,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": -0.042304106056690216,
+      "logits/rejected": 0.019032040610909462,
+      "logps/chosen": -1.4489058256149292,
+      "logps/rejected": -1.7134273052215576,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4489058256149292,
+      "rewards/margins": 0.2645213305950165,
+      "rewards/rejected": -1.7134273052215576,
+      "sft_loss": 1.475355625152588,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 5.317094749287823,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": -0.1466391235589981,
+      "logits/rejected": 0.12817367911338806,
+      "logps/chosen": -1.4559046030044556,
+      "logps/rejected": -1.783931016921997,
+      "loss": 0.492,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4559046030044556,
+      "rewards/margins": 0.32802626490592957,
+      "rewards/rejected": -1.783931016921997,
+      "sft_loss": 1.5369385480880737,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 3.8154740406992573,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": -0.07074271142482758,
+      "logits/rejected": 0.09242469817399979,
+      "logps/chosen": -1.4736456871032715,
+      "logps/rejected": -1.7548481225967407,
+      "loss": 0.5259,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4736456871032715,
+      "rewards/margins": 0.28120237588882446,
+      "rewards/rejected": -1.7548481225967407,
+      "sft_loss": 1.4908063411712646,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 5.561914424996803,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": -0.15970030426979065,
+      "logits/rejected": -0.026617711409926414,
+      "logps/chosen": -1.450312852859497,
+      "logps/rejected": -1.7166593074798584,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.450312852859497,
+      "rewards/margins": 0.26634639501571655,
+      "rewards/rejected": -1.7166593074798584,
+      "sft_loss": 1.5004311800003052,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 4.962043417804353,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": -0.10115758329629898,
+      "logits/rejected": -0.015957407653331757,
+      "logps/chosen": -1.4378993511199951,
+      "logps/rejected": -1.6727068424224854,
+      "loss": 0.532,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4378993511199951,
+      "rewards/margins": 0.23480752110481262,
+      "rewards/rejected": -1.6727068424224854,
+      "sft_loss": 1.531611442565918,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 4.892120380898045,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": -0.14030539989471436,
+      "logits/rejected": -0.05974206328392029,
+      "logps/chosen": -1.324191689491272,
+      "logps/rejected": -1.6707344055175781,
+      "loss": 0.4798,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.324191689491272,
+      "rewards/margins": 0.346542626619339,
+      "rewards/rejected": -1.6707344055175781,
+      "sft_loss": 1.3921024799346924,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 7.148324764965389,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": -0.2657248377799988,
+      "logits/rejected": -0.05455995723605156,
+      "logps/chosen": -1.4857937097549438,
+      "logps/rejected": -1.766405701637268,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4857937097549438,
+      "rewards/margins": 0.280612051486969,
+      "rewards/rejected": -1.766405701637268,
+      "sft_loss": 1.5520381927490234,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 7.551954007824476,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": -0.13647429645061493,
+      "logits/rejected": 0.00663360208272934,
+      "logps/chosen": -1.36500084400177,
+      "logps/rejected": -1.7560393810272217,
+      "loss": 0.4665,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.36500084400177,
+      "rewards/margins": 0.391038715839386,
+      "rewards/rejected": -1.7560393810272217,
+      "sft_loss": 1.4452471733093262,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 6.0992843581974405,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": -0.19032074511051178,
+      "logits/rejected": 0.009153047576546669,
+      "logps/chosen": -1.4787334203720093,
+      "logps/rejected": -1.8719135522842407,
+      "loss": 0.4813,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4787334203720093,
+      "rewards/margins": 0.39318010210990906,
+      "rewards/rejected": -1.8719135522842407,
+      "sft_loss": 1.5212489366531372,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 5.350166802718159,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": -0.0966397374868393,
+      "logits/rejected": 0.03161340951919556,
+      "logps/chosen": -1.4548650979995728,
+      "logps/rejected": -1.8944162130355835,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4548650979995728,
+      "rewards/margins": 0.4395512640476227,
+      "rewards/rejected": -1.8944162130355835,
+      "sft_loss": 1.4918707609176636,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 4.2595610579632615,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": -0.20996013283729553,
+      "logits/rejected": -0.03253777697682381,
+      "logps/chosen": -1.4889169931411743,
+      "logps/rejected": -2.000814437866211,
+      "loss": 0.4763,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4889169931411743,
+      "rewards/margins": 0.5118975639343262,
+      "rewards/rejected": -2.000814437866211,
+      "sft_loss": 1.5587809085845947,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 6.367327140932028,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": -0.12788192927837372,
+      "logits/rejected": 0.09394869208335876,
+      "logps/chosen": -1.5442017316818237,
+      "logps/rejected": -1.9254404306411743,
+      "loss": 0.5055,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5442017316818237,
+      "rewards/margins": 0.38123857975006104,
+      "rewards/rejected": -1.9254404306411743,
+      "sft_loss": 1.6286182403564453,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 4.895652101621527,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": -0.11432129144668579,
+      "logits/rejected": -0.04440194368362427,
+      "logps/chosen": -1.4022363424301147,
+      "logps/rejected": -1.7887109518051147,
+      "loss": 0.4738,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4022363424301147,
+      "rewards/margins": 0.3864745497703552,
+      "rewards/rejected": -1.7887109518051147,
+      "sft_loss": 1.4586220979690552,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 5.74006277179219,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": -0.18158350884914398,
+      "logits/rejected": -0.14121797680854797,
+      "logps/chosen": -1.4819562435150146,
+      "logps/rejected": -1.8232675790786743,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4819562435150146,
+      "rewards/margins": 0.3413112759590149,
+      "rewards/rejected": -1.8232675790786743,
+      "sft_loss": 1.5612026453018188,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 5.81208081953113,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": -0.0955495834350586,
+      "logits/rejected": -0.02487076446413994,
+      "logps/chosen": -1.397090196609497,
+      "logps/rejected": -1.9855940341949463,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.397090196609497,
+      "rewards/margins": 0.588503897190094,
+      "rewards/rejected": -1.9855940341949463,
+      "sft_loss": 1.4849728345870972,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 4.5388641581334355,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": -0.13712339103221893,
+      "logits/rejected": 0.014775209128856659,
+      "logps/chosen": -1.4033243656158447,
+      "logps/rejected": -1.7691576480865479,
+      "loss": 0.4902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4033243656158447,
+      "rewards/margins": 0.36583322286605835,
+      "rewards/rejected": -1.7691576480865479,
+      "sft_loss": 1.4541233777999878,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 6.767756108003455,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": -0.05116777494549751,
+      "logits/rejected": 0.04594787210226059,
+      "logps/chosen": -1.345689296722412,
+      "logps/rejected": -1.7165015935897827,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.345689296722412,
+      "rewards/margins": 0.37081241607666016,
+      "rewards/rejected": -1.7165015935897827,
+      "sft_loss": 1.4144160747528076,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 5.729373646531792,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": -0.02618979476392269,
+      "logits/rejected": 0.10099951922893524,
+      "logps/chosen": -1.4654579162597656,
+      "logps/rejected": -1.8436918258666992,
+      "loss": 0.5143,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4654579162597656,
+      "rewards/margins": 0.37823402881622314,
+      "rewards/rejected": -1.8436918258666992,
+      "sft_loss": 1.4997092485427856,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 6.710935394624452,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": -0.09092311561107635,
+      "logits/rejected": 0.07339204847812653,
+      "logps/chosen": -1.4836839437484741,
+      "logps/rejected": -1.7372486591339111,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4836839437484741,
+      "rewards/margins": 0.2535645067691803,
+      "rewards/rejected": -1.7372486591339111,
+      "sft_loss": 1.4888941049575806,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 7.69503470863718,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": -0.12186932563781738,
+      "logits/rejected": -0.016661647707223892,
+      "logps/chosen": -1.4644496440887451,
+      "logps/rejected": -1.7583051919937134,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4644496440887451,
+      "rewards/margins": 0.29385560750961304,
+      "rewards/rejected": -1.7583051919937134,
+      "sft_loss": 1.4940369129180908,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 4.511316509520965,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": -0.00946495495736599,
+      "logits/rejected": 0.15335066616535187,
+      "logps/chosen": -1.3617419004440308,
+      "logps/rejected": -1.8576990365982056,
+      "loss": 0.4515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3617419004440308,
+      "rewards/margins": 0.4959571957588196,
+      "rewards/rejected": -1.8576990365982056,
+      "sft_loss": 1.4200398921966553,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 8.821228426339408,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": -0.1066073551774025,
+      "logits/rejected": 0.062150947749614716,
+      "logps/chosen": -1.5116338729858398,
+      "logps/rejected": -1.8206783533096313,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5116338729858398,
+      "rewards/margins": 0.30904462933540344,
+      "rewards/rejected": -1.8206783533096313,
+      "sft_loss": 1.5308339595794678,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": 0.21312709152698517,
+      "eval_logits/rejected": 0.3106827437877655,
+      "eval_logps/chosen": -1.433026909828186,
+      "eval_logps/rejected": -1.807220697402954,
+      "eval_loss": 0.5046906471252441,
+      "eval_rewards/accuracies": 0.6305637955665588,
+      "eval_rewards/chosen": -1.433026909828186,
+      "eval_rewards/margins": 0.3741937279701233,
+      "eval_rewards/rejected": -1.807220697402954,
+      "eval_runtime": 43.4327,
+      "eval_samples_per_second": 30.967,
+      "eval_sft_loss": 1.488387107849121,
+      "eval_steps_per_second": 7.759,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 6.087055779041788,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": -0.06976554542779922,
+      "logits/rejected": 0.0286190714687109,
+      "logps/chosen": -1.464482069015503,
+      "logps/rejected": -1.7185428142547607,
+      "loss": 0.55,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.464482069015503,
+      "rewards/margins": 0.25406089425086975,
+      "rewards/rejected": -1.7185428142547607,
+      "sft_loss": 1.5256202220916748,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 6.057486499113076,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": -0.018949469551444054,
+      "logits/rejected": 0.09878715872764587,
+      "logps/chosen": -1.3560127019882202,
+      "logps/rejected": -1.6560802459716797,
+      "loss": 0.493,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3560127019882202,
+      "rewards/margins": 0.3000675141811371,
+      "rewards/rejected": -1.6560802459716797,
+      "sft_loss": 1.4456104040145874,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 7.26309234115051,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": -0.08260910212993622,
+      "logits/rejected": 0.07246457785367966,
+      "logps/chosen": -1.3358352184295654,
+      "logps/rejected": -1.6958932876586914,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3358352184295654,
+      "rewards/margins": 0.3600581884384155,
+      "rewards/rejected": -1.6958932876586914,
+      "sft_loss": 1.35738205909729,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 5.049206973817521,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": -0.15729498863220215,
+      "logits/rejected": 0.04396899417042732,
+      "logps/chosen": -1.4487788677215576,
+      "logps/rejected": -1.8310155868530273,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4487788677215576,
+      "rewards/margins": 0.38223665952682495,
+      "rewards/rejected": -1.8310155868530273,
+      "sft_loss": 1.454693078994751,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 5.494973211998128,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": -0.03109714947640896,
+      "logits/rejected": 0.11418769508600235,
+      "logps/chosen": -1.4256064891815186,
+      "logps/rejected": -1.8667850494384766,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4256064891815186,
+      "rewards/margins": 0.4411783218383789,
+      "rewards/rejected": -1.8667850494384766,
+      "sft_loss": 1.5188822746276855,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 4.978243268370478,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": -0.12496711313724518,
+      "logits/rejected": -0.09452895820140839,
+      "logps/chosen": -1.3800010681152344,
+      "logps/rejected": -1.6710617542266846,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3800010681152344,
+      "rewards/margins": 0.29106056690216064,
+      "rewards/rejected": -1.6710617542266846,
+      "sft_loss": 1.462308645248413,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 7.69197040790886,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": -0.1336095780134201,
+      "logits/rejected": 0.051111944019794464,
+      "logps/chosen": -1.4296190738677979,
+      "logps/rejected": -1.8351964950561523,
+      "loss": 0.505,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4296190738677979,
+      "rewards/margins": 0.4055773615837097,
+      "rewards/rejected": -1.8351964950561523,
+      "sft_loss": 1.469069242477417,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 33.53833279342883,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": -0.21496620774269104,
+      "logits/rejected": -0.13631033897399902,
+      "logps/chosen": -1.4748858213424683,
+      "logps/rejected": -1.791357398033142,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4748858213424683,
+      "rewards/margins": 0.3164716958999634,
+      "rewards/rejected": -1.791357398033142,
+      "sft_loss": 1.5261235237121582,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 7.422760096027858,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": -0.07252940535545349,
+      "logits/rejected": -0.029904400929808617,
+      "logps/chosen": -1.4626128673553467,
+      "logps/rejected": -1.828600525856018,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4626128673553467,
+      "rewards/margins": 0.3659875988960266,
+      "rewards/rejected": -1.828600525856018,
+      "sft_loss": 1.4483476877212524,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 6.7260498381113125,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": -0.19608174264431,
+      "logits/rejected": -0.0628303661942482,
+      "logps/chosen": -1.4419586658477783,
+      "logps/rejected": -1.7460864782333374,
+      "loss": 0.5296,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4419586658477783,
+      "rewards/margins": 0.3041277527809143,
+      "rewards/rejected": -1.7460864782333374,
+      "sft_loss": 1.54558527469635,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 5.623724890193116,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": -0.126641184091568,
+      "logits/rejected": 0.009659910574555397,
+      "logps/chosen": -1.4176418781280518,
+      "logps/rejected": -1.8081547021865845,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4176418781280518,
+      "rewards/margins": 0.39051297307014465,
+      "rewards/rejected": -1.8081547021865845,
+      "sft_loss": 1.5018881559371948,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 5.426254467551804,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": -0.08474062383174896,
+      "logits/rejected": -0.029642928391695023,
+      "logps/chosen": -1.4769197702407837,
+      "logps/rejected": -1.9251235723495483,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4769197702407837,
+      "rewards/margins": 0.4482037127017975,
+      "rewards/rejected": -1.9251235723495483,
+      "sft_loss": 1.5756006240844727,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 5.875803853615592,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": -0.11285390704870224,
+      "logits/rejected": 0.02270979806780815,
+      "logps/chosen": -1.445842981338501,
+      "logps/rejected": -1.824823021888733,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.445842981338501,
+      "rewards/margins": 0.37897998094558716,
+      "rewards/rejected": -1.824823021888733,
+      "sft_loss": 1.4722983837127686,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 7.160997591730322,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": -0.15063437819480896,
+      "logits/rejected": -0.0021080286242067814,
+      "logps/chosen": -1.3538507223129272,
+      "logps/rejected": -1.8220882415771484,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3538507223129272,
+      "rewards/margins": 0.4682373106479645,
+      "rewards/rejected": -1.8220882415771484,
+      "sft_loss": 1.4053022861480713,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 7.926408403181835,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": -0.16424348950386047,
+      "logits/rejected": -0.05737115070223808,
+      "logps/chosen": -1.4753576517105103,
+      "logps/rejected": -1.9165945053100586,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4753576517105103,
+      "rewards/margins": 0.44123712182044983,
+      "rewards/rejected": -1.9165945053100586,
+      "sft_loss": 1.5894849300384521,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 5.00403006548638,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": -0.23114259541034698,
+      "logits/rejected": -0.07610608637332916,
+      "logps/chosen": -1.3857789039611816,
+      "logps/rejected": -1.8070179224014282,
+      "loss": 0.4581,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3857789039611816,
+      "rewards/margins": 0.4212391972541809,
+      "rewards/rejected": -1.8070179224014282,
+      "sft_loss": 1.4718667268753052,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 9.039232195890655,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": -0.18809795379638672,
+      "logits/rejected": -0.07240627706050873,
+      "logps/chosen": -1.4653651714324951,
+      "logps/rejected": -1.8892971277236938,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4653651714324951,
+      "rewards/margins": 0.4239319860935211,
+      "rewards/rejected": -1.8892971277236938,
+      "sft_loss": 1.5039739608764648,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 6.0073439848626835,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": -0.2471090853214264,
+      "logits/rejected": -0.23510317504405975,
+      "logps/chosen": -1.439173698425293,
+      "logps/rejected": -1.6933097839355469,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.439173698425293,
+      "rewards/margins": 0.2541360557079315,
+      "rewards/rejected": -1.6933097839355469,
+      "sft_loss": 1.6191877126693726,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 6.183660125167922,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": -0.1707983911037445,
+      "logits/rejected": -0.020055923610925674,
+      "logps/chosen": -1.6324489116668701,
+      "logps/rejected": -1.9510940313339233,
+      "loss": 0.5457,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6324489116668701,
+      "rewards/margins": 0.31864503026008606,
+      "rewards/rejected": -1.9510940313339233,
+      "sft_loss": 1.6711390018463135,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 7.045846864225031,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": -0.08161846548318863,
+      "logits/rejected": -0.026478594169020653,
+      "logps/chosen": -1.6238008737564087,
+      "logps/rejected": -1.9390065670013428,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6238008737564087,
+      "rewards/margins": 0.31520575284957886,
+      "rewards/rejected": -1.9390065670013428,
+      "sft_loss": 1.637861967086792,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 8.244391342508212,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": -0.06960280239582062,
+      "logits/rejected": 0.05846409872174263,
+      "logps/chosen": -1.513601303100586,
+      "logps/rejected": -1.9503761529922485,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.513601303100586,
+      "rewards/margins": 0.4367748200893402,
+      "rewards/rejected": -1.9503761529922485,
+      "sft_loss": 1.5123342275619507,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 9.673424557945967,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": -0.18066242337226868,
+      "logits/rejected": -0.017239436507225037,
+      "logps/chosen": -1.4642972946166992,
+      "logps/rejected": -2.0365207195281982,
+      "loss": 0.49,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4642972946166992,
+      "rewards/margins": 0.5722236633300781,
+      "rewards/rejected": -2.0365207195281982,
+      "sft_loss": 1.5431797504425049,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 5.776090992074557,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": -0.023362573236227036,
+      "logits/rejected": 0.043627046048641205,
+      "logps/chosen": -1.438981294631958,
+      "logps/rejected": -1.8212578296661377,
+      "loss": 0.5045,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.438981294631958,
+      "rewards/margins": 0.3822764754295349,
+      "rewards/rejected": -1.8212578296661377,
+      "sft_loss": 1.5206745862960815,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 6.004133174670174,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": -0.16063198447227478,
+      "logits/rejected": -0.11137296259403229,
+      "logps/chosen": -1.452800989151001,
+      "logps/rejected": -1.7701566219329834,
+      "loss": 0.5031,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.452800989151001,
+      "rewards/margins": 0.3173556923866272,
+      "rewards/rejected": -1.7701566219329834,
+      "sft_loss": 1.5503771305084229,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 5.022632822038148,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": -0.12830975651741028,
+      "logits/rejected": -0.06951282173395157,
+      "logps/chosen": -1.4189093112945557,
+      "logps/rejected": -1.8615138530731201,
+      "loss": 0.4625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4189093112945557,
+      "rewards/margins": 0.4426047205924988,
+      "rewards/rejected": -1.8615138530731201,
+      "sft_loss": 1.4784224033355713,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 8.589342131042367,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": -0.09218287467956543,
+      "logits/rejected": 0.16176007688045502,
+      "logps/chosen": -1.4292819499969482,
+      "logps/rejected": -1.8710426092147827,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4292819499969482,
+      "rewards/margins": 0.44176048040390015,
+      "rewards/rejected": -1.8710426092147827,
+      "sft_loss": 1.517459750175476,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 4.69038184236173,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": -0.07629652321338654,
+      "logits/rejected": 0.09444563090801239,
+      "logps/chosen": -1.4645318984985352,
+      "logps/rejected": -1.9413913488388062,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4645318984985352,
+      "rewards/margins": 0.47685950994491577,
+      "rewards/rejected": -1.9413913488388062,
+      "sft_loss": 1.5025291442871094,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 9.283361039972426,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": -0.10875393450260162,
+      "logits/rejected": 0.03513630852103233,
+      "logps/chosen": -1.4688737392425537,
+      "logps/rejected": -1.8724868297576904,
+      "loss": 0.5058,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4688737392425537,
+      "rewards/margins": 0.40361303091049194,
+      "rewards/rejected": -1.8724868297576904,
+      "sft_loss": 1.504885196685791,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 6.486958122860878,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": -0.07364483177661896,
+      "logits/rejected": 0.024317726492881775,
+      "logps/chosen": -1.3688395023345947,
+      "logps/rejected": -1.88224196434021,
+      "loss": 0.457,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3688395023345947,
+      "rewards/margins": 0.5134025812149048,
+      "rewards/rejected": -1.88224196434021,
+      "sft_loss": 1.4580392837524414,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 5.235101078670038,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": -0.14383387565612793,
+      "logits/rejected": -0.08119402080774307,
+      "logps/chosen": -1.5173956155776978,
+      "logps/rejected": -1.9180179834365845,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5173956155776978,
+      "rewards/margins": 0.40062230825424194,
+      "rewards/rejected": -1.9180179834365845,
+      "sft_loss": 1.5680986642837524,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 6.894203466794916,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": -0.06564654409885406,
+      "logits/rejected": 0.009191582910716534,
+      "logps/chosen": -1.5647809505462646,
+      "logps/rejected": -1.9199464321136475,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5647809505462646,
+      "rewards/margins": 0.35516566038131714,
+      "rewards/rejected": -1.9199464321136475,
+      "sft_loss": 1.6604130268096924,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 7.756494004043538,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": -0.15478457510471344,
+      "logits/rejected": -0.04507964476943016,
+      "logps/chosen": -1.4633822441101074,
+      "logps/rejected": -1.9364042282104492,
+      "loss": 0.4843,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4633822441101074,
+      "rewards/margins": 0.47302207350730896,
+      "rewards/rejected": -1.9364042282104492,
+      "sft_loss": 1.5360000133514404,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 5.616002529498294,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": -0.1120419129729271,
+      "logits/rejected": 0.08104712516069412,
+      "logps/chosen": -1.493404507637024,
+      "logps/rejected": -1.9760732650756836,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.493404507637024,
+      "rewards/margins": 0.4826684594154358,
+      "rewards/rejected": -1.9760732650756836,
+      "sft_loss": 1.5499591827392578,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 4.46886017298503,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": -0.1037856787443161,
+      "logits/rejected": 0.07713104784488678,
+      "logps/chosen": -1.4457694292068481,
+      "logps/rejected": -1.9311643838882446,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4457694292068481,
+      "rewards/margins": 0.4853949546813965,
+      "rewards/rejected": -1.9311643838882446,
+      "sft_loss": 1.5152101516723633,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 4.794226804069966,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": -0.12479491531848907,
+      "logits/rejected": 0.06079835817217827,
+      "logps/chosen": -1.502943754196167,
+      "logps/rejected": -1.8639252185821533,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.502943754196167,
+      "rewards/margins": 0.36098119616508484,
+      "rewards/rejected": -1.8639252185821533,
+      "sft_loss": 1.5932748317718506,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 4.57957758743556,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": -0.13211789727210999,
+      "logits/rejected": -0.04276316240429878,
+      "logps/chosen": -1.4503905773162842,
+      "logps/rejected": -1.8833653926849365,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4503905773162842,
+      "rewards/margins": 0.432974636554718,
+      "rewards/rejected": -1.8833653926849365,
+      "sft_loss": 1.5527359247207642,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 7.278664056579846,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": -0.057872939854860306,
+      "logits/rejected": 0.09013696759939194,
+      "logps/chosen": -1.4882774353027344,
+      "logps/rejected": -1.8309959173202515,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4882774353027344,
+      "rewards/margins": 0.34271863102912903,
+      "rewards/rejected": -1.8309959173202515,
+      "sft_loss": 1.5207953453063965,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 5.104389335074668,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": 0.041389111429452896,
+      "logits/rejected": 0.10888870805501938,
+      "logps/chosen": -1.471793293952942,
+      "logps/rejected": -1.8024581670761108,
+      "loss": 0.5159,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.471793293952942,
+      "rewards/margins": 0.33066508173942566,
+      "rewards/rejected": -1.8024581670761108,
+      "sft_loss": 1.6215784549713135,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 6.218059706938761,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": -0.06100373715162277,
+      "logits/rejected": 0.03840227052569389,
+      "logps/chosen": -1.4952716827392578,
+      "logps/rejected": -1.9705145359039307,
+      "loss": 0.482,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4952716827392578,
+      "rewards/margins": 0.47524294257164,
+      "rewards/rejected": -1.9705145359039307,
+      "sft_loss": 1.6023286581039429,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 6.667214648831752,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": -0.15195073187351227,
+      "logits/rejected": -0.15606653690338135,
+      "logps/chosen": -1.4973087310791016,
+      "logps/rejected": -1.866207480430603,
+      "loss": 0.5178,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4973087310791016,
+      "rewards/margins": 0.368898868560791,
+      "rewards/rejected": -1.866207480430603,
+      "sft_loss": 1.6012775897979736,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 4.400604592157586,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": -0.17750287055969238,
+      "logits/rejected": 0.00775283295661211,
+      "logps/chosen": -1.3903168439865112,
+      "logps/rejected": -1.8271557092666626,
+      "loss": 0.4611,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3903168439865112,
+      "rewards/margins": 0.43683862686157227,
+      "rewards/rejected": -1.8271557092666626,
+      "sft_loss": 1.4089605808258057,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 5.489357722594127,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": -0.2346876859664917,
+      "logits/rejected": -0.10219166427850723,
+      "logps/chosen": -1.3173081874847412,
+      "logps/rejected": -1.856581687927246,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3173081874847412,
+      "rewards/margins": 0.5392736196517944,
+      "rewards/rejected": -1.856581687927246,
+      "sft_loss": 1.402467966079712,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 8.669951353681238,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": -0.18713152408599854,
+      "logits/rejected": -0.09777853637933731,
+      "logps/chosen": -1.5724120140075684,
+      "logps/rejected": -2.0409770011901855,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5724120140075684,
+      "rewards/margins": 0.4685649275779724,
+      "rewards/rejected": -2.0409770011901855,
+      "sft_loss": 1.5465257167816162,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 7.3927574477974645,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": -0.23059293627738953,
+      "logits/rejected": -0.09404326975345612,
+      "logps/chosen": -1.4030895233154297,
+      "logps/rejected": -1.7947998046875,
+      "loss": 0.5086,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4030895233154297,
+      "rewards/margins": 0.3917103409767151,
+      "rewards/rejected": -1.7947998046875,
+      "sft_loss": 1.4927089214324951,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 7.335250225773767,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": -0.22742590308189392,
+      "logits/rejected": -0.016776692122220993,
+      "logps/chosen": -1.3784120082855225,
+      "logps/rejected": -1.9083884954452515,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3784120082855225,
+      "rewards/margins": 0.5299763679504395,
+      "rewards/rejected": -1.9083884954452515,
+      "sft_loss": 1.474208950996399,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 8.880089590181422,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": -0.20148631930351257,
+      "logits/rejected": -0.04945765435695648,
+      "logps/chosen": -1.472278356552124,
+      "logps/rejected": -2.0694618225097656,
+      "loss": 0.4684,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.472278356552124,
+      "rewards/margins": 0.5971832871437073,
+      "rewards/rejected": -2.0694618225097656,
+      "sft_loss": 1.5050463676452637,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 6.805808478371301,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": -0.16491694748401642,
+      "logits/rejected": 0.033459730446338654,
+      "logps/chosen": -1.4134385585784912,
+      "logps/rejected": -1.9295330047607422,
+      "loss": 0.479,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4134385585784912,
+      "rewards/margins": 0.5160945057868958,
+      "rewards/rejected": -1.9295330047607422,
+      "sft_loss": 1.4623357057571411,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 5.422856370689168,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": -0.1376948058605194,
+      "logits/rejected": 0.04011141508817673,
+      "logps/chosen": -1.4443703889846802,
+      "logps/rejected": -1.9348556995391846,
+      "loss": 0.4503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4443703889846802,
+      "rewards/margins": 0.490485280752182,
+      "rewards/rejected": -1.9348556995391846,
+      "sft_loss": 1.5311527252197266,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 6.783491099675574,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": -0.10291273891925812,
+      "logits/rejected": -0.04401315003633499,
+      "logps/chosen": -1.4417774677276611,
+      "logps/rejected": -1.8393230438232422,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4417774677276611,
+      "rewards/margins": 0.39754557609558105,
+      "rewards/rejected": -1.8393230438232422,
+      "sft_loss": 1.530505657196045,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 7.2681706027953625,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": -0.06046579033136368,
+      "logits/rejected": -0.036283593624830246,
+      "logps/chosen": -1.4516606330871582,
+      "logps/rejected": -1.9776906967163086,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4516606330871582,
+      "rewards/margins": 0.5260300636291504,
+      "rewards/rejected": -1.9776906967163086,
+      "sft_loss": 1.5176465511322021,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 8.607018105014358,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": -0.13837631046772003,
+      "logits/rejected": -0.025722693651914597,
+      "logps/chosen": -1.5705254077911377,
+      "logps/rejected": -1.8737900257110596,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5705254077911377,
+      "rewards/margins": 0.3032645881175995,
+      "rewards/rejected": -1.8737900257110596,
+      "sft_loss": 1.5998989343643188,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 6.195684795946535,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": -0.08365651220083237,
+      "logits/rejected": -0.0891302078962326,
+      "logps/chosen": -1.462726354598999,
+      "logps/rejected": -1.742959976196289,
+      "loss": 0.5299,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.462726354598999,
+      "rewards/margins": 0.2802334725856781,
+      "rewards/rejected": -1.742959976196289,
+      "sft_loss": 1.5141435861587524,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 8.866576424845162,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": -0.18240386247634888,
+      "logits/rejected": -0.07936345040798187,
+      "logps/chosen": -1.5119251012802124,
+      "logps/rejected": -1.9127542972564697,
+      "loss": 0.5063,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5119251012802124,
+      "rewards/margins": 0.400829017162323,
+      "rewards/rejected": -1.9127542972564697,
+      "sft_loss": 1.6090915203094482,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 9.73327657484205,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": -0.26718050241470337,
+      "logits/rejected": -0.0880444124341011,
+      "logps/chosen": -1.5113370418548584,
+      "logps/rejected": -2.0135390758514404,
+      "loss": 0.491,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5113370418548584,
+      "rewards/margins": 0.5022020936012268,
+      "rewards/rejected": -2.0135390758514404,
+      "sft_loss": 1.5244777202606201,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 7.068729022101841,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": -0.14537358283996582,
+      "logits/rejected": -0.008987700566649437,
+      "logps/chosen": -1.4155831336975098,
+      "logps/rejected": -1.9859249591827393,
+      "loss": 0.4504,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4155831336975098,
+      "rewards/margins": 0.5703418850898743,
+      "rewards/rejected": -1.9859249591827393,
+      "sft_loss": 1.4823784828186035,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 10.14119788003141,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": -0.20737072825431824,
+      "logits/rejected": -0.033312804996967316,
+      "logps/chosen": -1.3519184589385986,
+      "logps/rejected": -1.7951476573944092,
+      "loss": 0.4764,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.3519184589385986,
+      "rewards/margins": 0.44322913885116577,
+      "rewards/rejected": -1.7951476573944092,
+      "sft_loss": 1.4460406303405762,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 5.044558530057508,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": -0.13140533864498138,
+      "logits/rejected": 0.07922891527414322,
+      "logps/chosen": -1.5612608194351196,
+      "logps/rejected": -2.098231315612793,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5612608194351196,
+      "rewards/margins": 0.5369703769683838,
+      "rewards/rejected": -2.098231315612793,
+      "sft_loss": 1.5844147205352783,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 5.122452335919795,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": -0.2452307939529419,
+      "logits/rejected": -0.07779756188392639,
+      "logps/chosen": -1.4014132022857666,
+      "logps/rejected": -1.8258777856826782,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4014132022857666,
+      "rewards/margins": 0.4244648516178131,
+      "rewards/rejected": -1.8258777856826782,
+      "sft_loss": 1.4864047765731812,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 6.735504262827155,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": -0.1678198277950287,
+      "logits/rejected": -0.07633207738399506,
+      "logps/chosen": -1.4897345304489136,
+      "logps/rejected": -1.9370830059051514,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4897345304489136,
+      "rewards/margins": 0.4473484456539154,
+      "rewards/rejected": -1.9370830059051514,
+      "sft_loss": 1.6406726837158203,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 9.594350265687709,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": -0.12510626018047333,
+      "logits/rejected": 0.01086263358592987,
+      "logps/chosen": -1.452544927597046,
+      "logps/rejected": -1.848858118057251,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.452544927597046,
+      "rewards/margins": 0.39631298184394836,
+      "rewards/rejected": -1.848858118057251,
+      "sft_loss": 1.5406442880630493,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 7.56576661024348,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": -0.15516726672649384,
+      "logits/rejected": -0.007723456714302301,
+      "logps/chosen": -1.6164741516113281,
+      "logps/rejected": -2.134988307952881,
+      "loss": 0.4919,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6164741516113281,
+      "rewards/margins": 0.5185142755508423,
+      "rewards/rejected": -2.134988307952881,
+      "sft_loss": 1.5903704166412354,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 7.056473066838253,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": -0.10760338604450226,
+      "logits/rejected": 0.16948586702346802,
+      "logps/chosen": -1.4854151010513306,
+      "logps/rejected": -2.0326244831085205,
+      "loss": 0.477,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4854151010513306,
+      "rewards/margins": 0.5472094416618347,
+      "rewards/rejected": -2.0326244831085205,
+      "sft_loss": 1.5109024047851562,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 6.750522317956841,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": -0.12459911406040192,
+      "logits/rejected": 0.06452270597219467,
+      "logps/chosen": -1.5161750316619873,
+      "logps/rejected": -2.0566658973693848,
+      "loss": 0.4867,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.5161750316619873,
+      "rewards/margins": 0.5404911637306213,
+      "rewards/rejected": -2.0566658973693848,
+      "sft_loss": 1.5886337757110596,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 5.275621529765781,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": -0.014655408449470997,
+      "logits/rejected": 0.020419131964445114,
+      "logps/chosen": -1.4924757480621338,
+      "logps/rejected": -1.9487247467041016,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4924757480621338,
+      "rewards/margins": 0.4562491476535797,
+      "rewards/rejected": -1.9487247467041016,
+      "sft_loss": 1.5474860668182373,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 6.810408731308987,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": 0.025302385911345482,
+      "logits/rejected": 0.11866162717342377,
+      "logps/chosen": -1.5684155225753784,
+      "logps/rejected": -2.068596601486206,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5684155225753784,
+      "rewards/margins": 0.5001809000968933,
+      "rewards/rejected": -2.068596601486206,
+      "sft_loss": 1.557191014289856,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 6.350633612711436,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": -0.1877928078174591,
+      "logits/rejected": -0.06601401418447495,
+      "logps/chosen": -1.4745948314666748,
+      "logps/rejected": -1.8821157217025757,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4745948314666748,
+      "rewards/margins": 0.4075208306312561,
+      "rewards/rejected": -1.8821157217025757,
+      "sft_loss": 1.5776586532592773,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 5.982336730467969,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": -0.020032238215208054,
+      "logits/rejected": 0.052525751292705536,
+      "logps/chosen": -1.539282202720642,
+      "logps/rejected": -2.0581915378570557,
+      "loss": 0.4677,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.539282202720642,
+      "rewards/margins": 0.5189090967178345,
+      "rewards/rejected": -2.0581915378570557,
+      "sft_loss": 1.6047948598861694,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 5.214865914779404,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": -0.19301816821098328,
+      "logits/rejected": 0.0031130120623856783,
+      "logps/chosen": -1.5542106628417969,
+      "logps/rejected": -1.9580436944961548,
+      "loss": 0.5151,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5542106628417969,
+      "rewards/margins": 0.40383315086364746,
+      "rewards/rejected": -1.9580436944961548,
+      "sft_loss": 1.6011720895767212,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 6.088334407434233,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": -0.11648712307214737,
+      "logits/rejected": 0.06812908500432968,
+      "logps/chosen": -1.4927353858947754,
+      "logps/rejected": -1.9419381618499756,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4927353858947754,
+      "rewards/margins": 0.4492027163505554,
+      "rewards/rejected": -1.9419381618499756,
+      "sft_loss": 1.5813872814178467,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 6.118015819779254,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": -0.12596911191940308,
+      "logits/rejected": 0.03608747571706772,
+      "logps/chosen": -1.5252940654754639,
+      "logps/rejected": -1.930368185043335,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5252940654754639,
+      "rewards/margins": 0.4050741195678711,
+      "rewards/rejected": -1.930368185043335,
+      "sft_loss": 1.5665839910507202,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 7.019082824371375,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": -0.04565399885177612,
+      "logits/rejected": 0.10724995285272598,
+      "logps/chosen": -1.3797297477722168,
+      "logps/rejected": -1.9839117527008057,
+      "loss": 0.4677,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3797297477722168,
+      "rewards/margins": 0.6041820049285889,
+      "rewards/rejected": -1.9839117527008057,
+      "sft_loss": 1.4846323728561401,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 9.39532993132623,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": -0.06291689723730087,
+      "logits/rejected": 0.09211830794811249,
+      "logps/chosen": -1.4899945259094238,
+      "logps/rejected": -2.1186699867248535,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4899945259094238,
+      "rewards/margins": 0.6286753416061401,
+      "rewards/rejected": -2.1186699867248535,
+      "sft_loss": 1.5955866575241089,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 4.337083396947069,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": -0.06913149356842041,
+      "logits/rejected": 0.07702628523111343,
+      "logps/chosen": -1.4909687042236328,
+      "logps/rejected": -2.0008749961853027,
+      "loss": 0.488,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4909687042236328,
+      "rewards/margins": 0.5099064111709595,
+      "rewards/rejected": -2.0008749961853027,
+      "sft_loss": 1.5813932418823242,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 9.719493267673636,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": -0.057274866849184036,
+      "logits/rejected": 0.08096132427453995,
+      "logps/chosen": -1.5333844423294067,
+      "logps/rejected": -1.9823888540267944,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5333844423294067,
+      "rewards/margins": 0.44900450110435486,
+      "rewards/rejected": -1.9823888540267944,
+      "sft_loss": 1.634751319885254,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 6.187677361284805,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": -0.1976046860218048,
+      "logits/rejected": -0.0066480934619903564,
+      "logps/chosen": -1.388067364692688,
+      "logps/rejected": -1.7951103448867798,
+      "loss": 0.4872,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.388067364692688,
+      "rewards/margins": 0.40704289078712463,
+      "rewards/rejected": -1.7951103448867798,
+      "sft_loss": 1.4659048318862915,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 10.546333717242193,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": -0.17652028799057007,
+      "logits/rejected": 0.04019765183329582,
+      "logps/chosen": -1.5461647510528564,
+      "logps/rejected": -2.09086275100708,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5461647510528564,
+      "rewards/margins": 0.5446978807449341,
+      "rewards/rejected": -2.09086275100708,
+      "sft_loss": 1.6310644149780273,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 5.631643384912899,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": -0.017530517652630806,
+      "logits/rejected": 0.10928189754486084,
+      "logps/chosen": -1.5052597522735596,
+      "logps/rejected": -1.9419084787368774,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5052597522735596,
+      "rewards/margins": 0.4366486072540283,
+      "rewards/rejected": -1.9419084787368774,
+      "sft_loss": 1.5173150300979614,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 5.330658404953745,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": -0.040447741746902466,
+      "logits/rejected": 0.03319546580314636,
+      "logps/chosen": -1.4307003021240234,
+      "logps/rejected": -2.0200963020324707,
+      "loss": 0.4367,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4307003021240234,
+      "rewards/margins": 0.5893961191177368,
+      "rewards/rejected": -2.0200963020324707,
+      "sft_loss": 1.532576560974121,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 6.503741181473453,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": -0.22894029319286346,
+      "logits/rejected": -0.08977676928043365,
+      "logps/chosen": -1.39363694190979,
+      "logps/rejected": -1.8419349193572998,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.39363694190979,
+      "rewards/margins": 0.44829821586608887,
+      "rewards/rejected": -1.8419349193572998,
+      "sft_loss": 1.4875110387802124,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 8.559354194639448,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": -0.08281628787517548,
+      "logits/rejected": 0.05056994408369064,
+      "logps/chosen": -1.4323084354400635,
+      "logps/rejected": -1.8499174118041992,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4323084354400635,
+      "rewards/margins": 0.41760897636413574,
+      "rewards/rejected": -1.8499174118041992,
+      "sft_loss": 1.5048401355743408,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": 0.21058468520641327,
+      "eval_logits/rejected": 0.3098817765712738,
+      "eval_logps/chosen": -1.5341204404830933,
+      "eval_logps/rejected": -2.0179104804992676,
+      "eval_loss": 0.49633318185806274,
+      "eval_rewards/accuracies": 0.6535608172416687,
+      "eval_rewards/chosen": -1.5341204404830933,
+      "eval_rewards/margins": 0.48379021883010864,
+      "eval_rewards/rejected": -2.0179104804992676,
+      "eval_runtime": 43.3901,
+      "eval_samples_per_second": 30.998,
+      "eval_sft_loss": 1.6007072925567627,
+      "eval_steps_per_second": 7.767,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 4.898881519182048,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": -0.06763359159231186,
+      "logits/rejected": 0.10140136629343033,
+      "logps/chosen": -1.5375685691833496,
+      "logps/rejected": -2.103701591491699,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5375685691833496,
+      "rewards/margins": 0.5661331415176392,
+      "rewards/rejected": -2.103701591491699,
+      "sft_loss": 1.5908316373825073,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 7.678119509091904,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": -0.20165202021598816,
+      "logits/rejected": 0.02417032979428768,
+      "logps/chosen": -1.4833492040634155,
+      "logps/rejected": -1.7696387767791748,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4833492040634155,
+      "rewards/margins": 0.28628966212272644,
+      "rewards/rejected": -1.7696387767791748,
+      "sft_loss": 1.563765525817871,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 5.865490300549153,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": -0.04911652207374573,
+      "logits/rejected": 0.03724956512451172,
+      "logps/chosen": -1.4184011220932007,
+      "logps/rejected": -2.0442159175872803,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4184011220932007,
+      "rewards/margins": 0.6258147954940796,
+      "rewards/rejected": -2.0442159175872803,
+      "sft_loss": 1.4617655277252197,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 5.003988637593825,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": -0.1761176586151123,
+      "logits/rejected": -0.02824026718735695,
+      "logps/chosen": -1.5366970300674438,
+      "logps/rejected": -2.046592950820923,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5366970300674438,
+      "rewards/margins": 0.509895920753479,
+      "rewards/rejected": -2.046592950820923,
+      "sft_loss": 1.6355535984039307,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 5.068686232645391,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": -0.064757339656353,
+      "logits/rejected": 0.01880965568125248,
+      "logps/chosen": -1.406187891960144,
+      "logps/rejected": -1.8038914203643799,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.406187891960144,
+      "rewards/margins": 0.3977034091949463,
+      "rewards/rejected": -1.8038914203643799,
+      "sft_loss": 1.528364896774292,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 15.350732764917854,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": -0.05523641034960747,
+      "logits/rejected": 0.01888253539800644,
+      "logps/chosen": -1.5876901149749756,
+      "logps/rejected": -2.023746967315674,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5876901149749756,
+      "rewards/margins": 0.436056911945343,
+      "rewards/rejected": -2.023746967315674,
+      "sft_loss": 1.6234697103500366,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 6.027884750611186,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": -0.04204241558909416,
+      "logits/rejected": 0.09753796458244324,
+      "logps/chosen": -1.4533779621124268,
+      "logps/rejected": -1.7768032550811768,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4533779621124268,
+      "rewards/margins": 0.3234253525733948,
+      "rewards/rejected": -1.7768032550811768,
+      "sft_loss": 1.493114948272705,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 7.734356836200579,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": -0.1577232927083969,
+      "logits/rejected": -0.027221685275435448,
+      "logps/chosen": -1.4389315843582153,
+      "logps/rejected": -1.936092734336853,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4389315843582153,
+      "rewards/margins": 0.49716120958328247,
+      "rewards/rejected": -1.936092734336853,
+      "sft_loss": 1.449273943901062,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 7.371662500469119,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": -0.10918200016021729,
+      "logits/rejected": 0.04994308948516846,
+      "logps/chosen": -1.4908276796340942,
+      "logps/rejected": -1.8749077320098877,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4908276796340942,
+      "rewards/margins": 0.38408005237579346,
+      "rewards/rejected": -1.8749077320098877,
+      "sft_loss": 1.526848554611206,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 7.61863988294174,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": -0.09959923475980759,
+      "logits/rejected": 0.06935176998376846,
+      "logps/chosen": -1.423628807067871,
+      "logps/rejected": -1.949601411819458,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.423628807067871,
+      "rewards/margins": 0.5259725451469421,
+      "rewards/rejected": -1.949601411819458,
+      "sft_loss": 1.4399412870407104,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 4.815212043827955,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": -0.1309075951576233,
+      "logits/rejected": 0.1330709159374237,
+      "logps/chosen": -1.513875126838684,
+      "logps/rejected": -1.957093596458435,
+      "loss": 0.4915,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.513875126838684,
+      "rewards/margins": 0.44321855902671814,
+      "rewards/rejected": -1.957093596458435,
+      "sft_loss": 1.5696543455123901,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 11.562215211111699,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": -0.060210179537534714,
+      "logits/rejected": 0.06136656925082207,
+      "logps/chosen": -1.4934711456298828,
+      "logps/rejected": -1.8130403757095337,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4934711456298828,
+      "rewards/margins": 0.31956934928894043,
+      "rewards/rejected": -1.8130403757095337,
+      "sft_loss": 1.5263965129852295,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 5.522447694900418,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": -0.06598939001560211,
+      "logits/rejected": 0.02520240843296051,
+      "logps/chosen": -1.4613018035888672,
+      "logps/rejected": -2.028432846069336,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4613018035888672,
+      "rewards/margins": 0.5671309232711792,
+      "rewards/rejected": -2.028432846069336,
+      "sft_loss": 1.480936884880066,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 6.621376942660697,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": -0.2126634120941162,
+      "logits/rejected": -0.06398321688175201,
+      "logps/chosen": -1.438204288482666,
+      "logps/rejected": -1.7354466915130615,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.438204288482666,
+      "rewards/margins": 0.2972424626350403,
+      "rewards/rejected": -1.7354466915130615,
+      "sft_loss": 1.5432885885238647,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 5.988214386261141,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": -0.07591654360294342,
+      "logits/rejected": -7.105171971488744e-05,
+      "logps/chosen": -1.4517724514007568,
+      "logps/rejected": -1.8585712909698486,
+      "loss": 0.4723,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4517724514007568,
+      "rewards/margins": 0.40679866075515747,
+      "rewards/rejected": -1.8585712909698486,
+      "sft_loss": 1.5348371267318726,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 6.05179775803104,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": -0.22602811455726624,
+      "logits/rejected": -0.03507426753640175,
+      "logps/chosen": -1.504457712173462,
+      "logps/rejected": -2.0625691413879395,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.504457712173462,
+      "rewards/margins": 0.5581115484237671,
+      "rewards/rejected": -2.0625691413879395,
+      "sft_loss": 1.6593469381332397,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 5.612677415801948,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": -0.04492194578051567,
+      "logits/rejected": 0.1510792225599289,
+      "logps/chosen": -1.5203931331634521,
+      "logps/rejected": -2.1091973781585693,
+      "loss": 0.505,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5203931331634521,
+      "rewards/margins": 0.588804304599762,
+      "rewards/rejected": -2.1091973781585693,
+      "sft_loss": 1.6557775735855103,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 8.054763097158059,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": -0.1551412045955658,
+      "logits/rejected": -0.09231224656105042,
+      "logps/chosen": -1.55299973487854,
+      "logps/rejected": -2.0047755241394043,
+      "loss": 0.4886,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.55299973487854,
+      "rewards/margins": 0.45177555084228516,
+      "rewards/rejected": -2.0047755241394043,
+      "sft_loss": 1.62527596950531,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 6.649605258874318,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": -0.03161294385790825,
+      "logits/rejected": -0.020646898075938225,
+      "logps/chosen": -1.4861562252044678,
+      "logps/rejected": -1.8771871328353882,
+      "loss": 0.5127,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4861562252044678,
+      "rewards/margins": 0.39103081822395325,
+      "rewards/rejected": -1.8771871328353882,
+      "sft_loss": 1.543869137763977,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 5.826374531410804,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": -0.06338343769311905,
+      "logits/rejected": 0.1107725128531456,
+      "logps/chosen": -1.4578975439071655,
+      "logps/rejected": -1.8605518341064453,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4578975439071655,
+      "rewards/margins": 0.402654230594635,
+      "rewards/rejected": -1.8605518341064453,
+      "sft_loss": 1.57229483127594,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 6.41371973658278,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": -0.04192394018173218,
+      "logits/rejected": 0.14050684869289398,
+      "logps/chosen": -1.3707352876663208,
+      "logps/rejected": -1.8369287252426147,
+      "loss": 0.4817,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.3707352876663208,
+      "rewards/margins": 0.4661934971809387,
+      "rewards/rejected": -1.8369287252426147,
+      "sft_loss": 1.4329276084899902,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 6.924270674766152,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": -0.1159440129995346,
+      "logits/rejected": 0.03930569440126419,
+      "logps/chosen": -1.4073395729064941,
+      "logps/rejected": -1.8557002544403076,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4073395729064941,
+      "rewards/margins": 0.44836077094078064,
+      "rewards/rejected": -1.8557002544403076,
+      "sft_loss": 1.4969781637191772,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 6.41722619481814,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": -0.047092996537685394,
+      "logits/rejected": 0.08643031120300293,
+      "logps/chosen": -1.4872676134109497,
+      "logps/rejected": -1.9126088619232178,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4872676134109497,
+      "rewards/margins": 0.4253414273262024,
+      "rewards/rejected": -1.9126088619232178,
+      "sft_loss": 1.61127507686615,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 7.916966014649849,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": -0.1546896994113922,
+      "logits/rejected": -0.06426952034235,
+      "logps/chosen": -1.4904658794403076,
+      "logps/rejected": -1.8696438074111938,
+      "loss": 0.4876,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4904658794403076,
+      "rewards/margins": 0.3791780471801758,
+      "rewards/rejected": -1.8696438074111938,
+      "sft_loss": 1.4804034233093262,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 8.077287376418578,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": -0.10523150116205215,
+      "logits/rejected": -0.01580960303544998,
+      "logps/chosen": -1.5716774463653564,
+      "logps/rejected": -2.037581205368042,
+      "loss": 0.4826,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5716774463653564,
+      "rewards/margins": 0.4659039080142975,
+      "rewards/rejected": -2.037581205368042,
+      "sft_loss": 1.7261673212051392,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 10.231820821782932,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": -0.1276179999113083,
+      "logits/rejected": -0.06458455324172974,
+      "logps/chosen": -1.5572798252105713,
+      "logps/rejected": -1.858924150466919,
+      "loss": 0.5626,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5572798252105713,
+      "rewards/margins": 0.3016444146633148,
+      "rewards/rejected": -1.858924150466919,
+      "sft_loss": 1.6395533084869385,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 4.578023625051732,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": -0.18592733144760132,
+      "logits/rejected": -0.018612224608659744,
+      "logps/chosen": -1.448899269104004,
+      "logps/rejected": -1.9447243213653564,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.448899269104004,
+      "rewards/margins": 0.49582499265670776,
+      "rewards/rejected": -1.9447243213653564,
+      "sft_loss": 1.5233696699142456,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 7.529452510480847,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": -0.1055329218506813,
+      "logits/rejected": 0.09222625941038132,
+      "logps/chosen": -1.5448589324951172,
+      "logps/rejected": -1.9362952709197998,
+      "loss": 0.514,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5448589324951172,
+      "rewards/margins": 0.39143624901771545,
+      "rewards/rejected": -1.9362952709197998,
+      "sft_loss": 1.6498651504516602,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 5.551868702561516,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": -0.08056094497442245,
+      "logits/rejected": 0.1106506809592247,
+      "logps/chosen": -1.4577360153198242,
+      "logps/rejected": -1.9888372421264648,
+      "loss": 0.4764,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4577360153198242,
+      "rewards/margins": 0.5311012268066406,
+      "rewards/rejected": -1.9888372421264648,
+      "sft_loss": 1.5364185571670532,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 5.945591994335276,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": -0.11514048278331757,
+      "logits/rejected": 0.06404171884059906,
+      "logps/chosen": -1.470406413078308,
+      "logps/rejected": -1.8387157917022705,
+      "loss": 0.501,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.470406413078308,
+      "rewards/margins": 0.36830934882164,
+      "rewards/rejected": -1.8387157917022705,
+      "sft_loss": 1.5712687969207764,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 12.049719645699561,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": -0.05989614874124527,
+      "logits/rejected": 0.039876788854599,
+      "logps/chosen": -1.381488561630249,
+      "logps/rejected": -1.919499158859253,
+      "loss": 0.4806,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.381488561630249,
+      "rewards/margins": 0.5380104780197144,
+      "rewards/rejected": -1.919499158859253,
+      "sft_loss": 1.464348316192627,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 10.506638498291947,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": -0.06798264384269714,
+      "logits/rejected": 0.07144947350025177,
+      "logps/chosen": -1.478407621383667,
+      "logps/rejected": -1.9259250164031982,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.478407621383667,
+      "rewards/margins": 0.4475174844264984,
+      "rewards/rejected": -1.9259250164031982,
+      "sft_loss": 1.546134352684021,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 7.133123060324439,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": -0.06579507887363434,
+      "logits/rejected": 0.0746840387582779,
+      "logps/chosen": -1.4398367404937744,
+      "logps/rejected": -2.0183796882629395,
+      "loss": 0.446,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4398367404937744,
+      "rewards/margins": 0.578542947769165,
+      "rewards/rejected": -2.0183796882629395,
+      "sft_loss": 1.5617057085037231,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 5.902788356834458,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": -0.0012567430967465043,
+      "logits/rejected": 0.07987087965011597,
+      "logps/chosen": -1.449086308479309,
+      "logps/rejected": -1.943660020828247,
+      "loss": 0.4738,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.449086308479309,
+      "rewards/margins": 0.49457383155822754,
+      "rewards/rejected": -1.943660020828247,
+      "sft_loss": 1.5337722301483154,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 6.881337573413881,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": -0.15774737298488617,
+      "logits/rejected": 0.0003899022995028645,
+      "logps/chosen": -1.416227102279663,
+      "logps/rejected": -1.8630616664886475,
+      "loss": 0.4742,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.416227102279663,
+      "rewards/margins": 0.4468347132205963,
+      "rewards/rejected": -1.8630616664886475,
+      "sft_loss": 1.4903398752212524,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 5.919829446263836,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": -0.0845089852809906,
+      "logits/rejected": 0.014165307395160198,
+      "logps/chosen": -1.5333517789840698,
+      "logps/rejected": -1.879889726638794,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5333517789840698,
+      "rewards/margins": 0.34653788805007935,
+      "rewards/rejected": -1.879889726638794,
+      "sft_loss": 1.6552213430404663,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 6.967963627355345,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": 0.04399329423904419,
+      "logits/rejected": 0.21903137862682343,
+      "logps/chosen": -1.3956737518310547,
+      "logps/rejected": -1.9111979007720947,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3956737518310547,
+      "rewards/margins": 0.5155242085456848,
+      "rewards/rejected": -1.9111979007720947,
+      "sft_loss": 1.4362618923187256,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 7.943512130584483,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": -0.08490066230297089,
+      "logits/rejected": 0.0671260803937912,
+      "logps/chosen": -1.4574105739593506,
+      "logps/rejected": -1.8720905780792236,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4574105739593506,
+      "rewards/margins": 0.4146800935268402,
+      "rewards/rejected": -1.8720905780792236,
+      "sft_loss": 1.611249327659607,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 5.7395767064625085,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": -0.0439167320728302,
+      "logits/rejected": 0.013581424951553345,
+      "logps/chosen": -1.4777257442474365,
+      "logps/rejected": -1.838610053062439,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4777257442474365,
+      "rewards/margins": 0.36088424921035767,
+      "rewards/rejected": -1.838610053062439,
+      "sft_loss": 1.5101115703582764,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 6.597324797319752,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": 0.04372371733188629,
+      "logits/rejected": 0.19077324867248535,
+      "logps/chosen": -1.4668786525726318,
+      "logps/rejected": -1.8124185800552368,
+      "loss": 0.507,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4668786525726318,
+      "rewards/margins": 0.34553974866867065,
+      "rewards/rejected": -1.8124185800552368,
+      "sft_loss": 1.5253021717071533,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 10.765397785323103,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": 0.017538107931613922,
+      "logits/rejected": 0.07946206629276276,
+      "logps/chosen": -1.453668236732483,
+      "logps/rejected": -1.8302783966064453,
+      "loss": 0.482,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.453668236732483,
+      "rewards/margins": 0.37661033868789673,
+      "rewards/rejected": -1.8302783966064453,
+      "sft_loss": 1.5746830701828003,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 10.862884100539372,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": 0.047561369836330414,
+      "logits/rejected": 0.13421495258808136,
+      "logps/chosen": -1.5060184001922607,
+      "logps/rejected": -1.8954282999038696,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5060184001922607,
+      "rewards/margins": 0.3894098699092865,
+      "rewards/rejected": -1.8954282999038696,
+      "sft_loss": 1.605099081993103,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 13.4304998996284,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": -0.08659394830465317,
+      "logits/rejected": 0.07501087337732315,
+      "logps/chosen": -1.3954126834869385,
+      "logps/rejected": -1.8897117376327515,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3954126834869385,
+      "rewards/margins": 0.49429893493652344,
+      "rewards/rejected": -1.8897117376327515,
+      "sft_loss": 1.5091125965118408,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 6.402499565877001,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": -0.0855594277381897,
+      "logits/rejected": -0.08354773372411728,
+      "logps/chosen": -1.53382408618927,
+      "logps/rejected": -1.7779781818389893,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.53382408618927,
+      "rewards/margins": 0.2441541850566864,
+      "rewards/rejected": -1.7779781818389893,
+      "sft_loss": 1.613317847251892,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 11.188739880849315,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": -0.018711065873503685,
+      "logits/rejected": 0.15435022115707397,
+      "logps/chosen": -1.4651854038238525,
+      "logps/rejected": -1.893734335899353,
+      "loss": 0.504,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4651854038238525,
+      "rewards/margins": 0.42854875326156616,
+      "rewards/rejected": -1.893734335899353,
+      "sft_loss": 1.4950653314590454,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 8.058799091808403,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": -0.1831529587507248,
+      "logits/rejected": 0.02995426021516323,
+      "logps/chosen": -1.525868535041809,
+      "logps/rejected": -1.9160089492797852,
+      "loss": 0.5117,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.525868535041809,
+      "rewards/margins": 0.3901405334472656,
+      "rewards/rejected": -1.9160089492797852,
+      "sft_loss": 1.5806645154953003,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 10.003547271148747,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": -0.0974530428647995,
+      "logits/rejected": -0.0012318312656134367,
+      "logps/chosen": -1.4436627626419067,
+      "logps/rejected": -1.8610448837280273,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4436627626419067,
+      "rewards/margins": 0.41738200187683105,
+      "rewards/rejected": -1.8610448837280273,
+      "sft_loss": 1.5641653537750244,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 4.95988489497464,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": -0.04262874275445938,
+      "logits/rejected": 0.05346344783902168,
+      "logps/chosen": -1.5752532482147217,
+      "logps/rejected": -1.998945951461792,
+      "loss": 0.507,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5752532482147217,
+      "rewards/margins": 0.4236927628517151,
+      "rewards/rejected": -1.998945951461792,
+      "sft_loss": 1.6537303924560547,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 9.02503655381388,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": -0.08658730983734131,
+      "logits/rejected": 0.009633921086788177,
+      "logps/chosen": -1.4610365629196167,
+      "logps/rejected": -1.9266433715820312,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4610365629196167,
+      "rewards/margins": 0.4656066298484802,
+      "rewards/rejected": -1.9266433715820312,
+      "sft_loss": 1.507159948348999,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 5.88129327721945,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": -0.17061766982078552,
+      "logits/rejected": -0.0011933178175240755,
+      "logps/chosen": -1.490103006362915,
+      "logps/rejected": -2.159802198410034,
+      "loss": 0.4592,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.490103006362915,
+      "rewards/margins": 0.6696991920471191,
+      "rewards/rejected": -2.159802198410034,
+      "sft_loss": 1.5680917501449585,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 7.396741228216022,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": -0.06351391226053238,
+      "logits/rejected": 0.1041162982583046,
+      "logps/chosen": -1.3809651136398315,
+      "logps/rejected": -1.8353707790374756,
+      "loss": 0.4499,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3809651136398315,
+      "rewards/margins": 0.45440563559532166,
+      "rewards/rejected": -1.8353707790374756,
+      "sft_loss": 1.4856082201004028,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 8.178732535368553,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": -0.004477344453334808,
+      "logits/rejected": 0.054392505437135696,
+      "logps/chosen": -1.3770705461502075,
+      "logps/rejected": -1.8496062755584717,
+      "loss": 0.4751,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3770705461502075,
+      "rewards/margins": 0.47253578901290894,
+      "rewards/rejected": -1.8496062755584717,
+      "sft_loss": 1.4481415748596191,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 6.186302174172172,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": -0.18965403735637665,
+      "logits/rejected": -0.05282963067293167,
+      "logps/chosen": -1.4871896505355835,
+      "logps/rejected": -1.9702796936035156,
+      "loss": 0.4646,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4871896505355835,
+      "rewards/margins": 0.4830901622772217,
+      "rewards/rejected": -1.9702796936035156,
+      "sft_loss": 1.6419436931610107,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 10.451953780852376,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": -0.04714593291282654,
+      "logits/rejected": 0.06453403830528259,
+      "logps/chosen": -1.6118723154067993,
+      "logps/rejected": -1.9500125646591187,
+      "loss": 0.531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6118723154067993,
+      "rewards/margins": 0.3381403386592865,
+      "rewards/rejected": -1.9500125646591187,
+      "sft_loss": 1.6536245346069336,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 8.909876566015209,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": -0.10542843490839005,
+      "logits/rejected": 0.0743652805685997,
+      "logps/chosen": -1.531923770904541,
+      "logps/rejected": -1.9045947790145874,
+      "loss": 0.548,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.531923770904541,
+      "rewards/margins": 0.3726711869239807,
+      "rewards/rejected": -1.9045947790145874,
+      "sft_loss": 1.4829033613204956,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 6.795360694229123,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": -0.0952523946762085,
+      "logits/rejected": 0.09471447020769119,
+      "logps/chosen": -1.4994065761566162,
+      "logps/rejected": -1.892115831375122,
+      "loss": 0.5021,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4994065761566162,
+      "rewards/margins": 0.3927091658115387,
+      "rewards/rejected": -1.892115831375122,
+      "sft_loss": 1.588990569114685,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 7.164348437941818,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": -0.10167787969112396,
+      "logits/rejected": 0.13535355031490326,
+      "logps/chosen": -1.445357084274292,
+      "logps/rejected": -1.8391374349594116,
+      "loss": 0.4784,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.445357084274292,
+      "rewards/margins": 0.3937804102897644,
+      "rewards/rejected": -1.8391374349594116,
+      "sft_loss": 1.4996330738067627,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 17.290840299053265,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": -0.22392210364341736,
+      "logits/rejected": 0.04952241852879524,
+      "logps/chosen": -1.4616634845733643,
+      "logps/rejected": -2.038735866546631,
+      "loss": 0.4518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4616634845733643,
+      "rewards/margins": 0.5770725011825562,
+      "rewards/rejected": -2.038735866546631,
+      "sft_loss": 1.5287370681762695,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 5.606463831178428,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": -0.021728316321969032,
+      "logits/rejected": 0.015017149038612843,
+      "logps/chosen": -1.4510247707366943,
+      "logps/rejected": -1.9158073663711548,
+      "loss": 0.493,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4510247707366943,
+      "rewards/margins": 0.4647824764251709,
+      "rewards/rejected": -1.9158073663711548,
+      "sft_loss": 1.5007692575454712,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 7.192489826017525,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": -0.1267596036195755,
+      "logits/rejected": 0.0693402960896492,
+      "logps/chosen": -1.5120747089385986,
+      "logps/rejected": -1.962651014328003,
+      "loss": 0.4876,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5120747089385986,
+      "rewards/margins": 0.45057615637779236,
+      "rewards/rejected": -1.962651014328003,
+      "sft_loss": 1.6224653720855713,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 5.613140069940458,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": -0.16893896460533142,
+      "logits/rejected": 0.06323234736919403,
+      "logps/chosen": -1.3807225227355957,
+      "logps/rejected": -1.9726345539093018,
+      "loss": 0.4286,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3807225227355957,
+      "rewards/margins": 0.5919119715690613,
+      "rewards/rejected": -1.9726345539093018,
+      "sft_loss": 1.4009912014007568,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 8.75005969074735,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": -0.02317567728459835,
+      "logits/rejected": 0.009615430608391762,
+      "logps/chosen": -1.4695868492126465,
+      "logps/rejected": -1.9241955280303955,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4695868492126465,
+      "rewards/margins": 0.45460858941078186,
+      "rewards/rejected": -1.9241955280303955,
+      "sft_loss": 1.501080870628357,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 4.506798373707267,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": -0.19773828983306885,
+      "logits/rejected": -0.07556317001581192,
+      "logps/chosen": -1.377198576927185,
+      "logps/rejected": -1.7756855487823486,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.377198576927185,
+      "rewards/margins": 0.3984869718551636,
+      "rewards/rejected": -1.7756855487823486,
+      "sft_loss": 1.522790789604187,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 7.052556075358587,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": -0.1464635580778122,
+      "logits/rejected": -0.015275077894330025,
+      "logps/chosen": -1.4300029277801514,
+      "logps/rejected": -1.7926452159881592,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4300029277801514,
+      "rewards/margins": 0.3626423180103302,
+      "rewards/rejected": -1.7926452159881592,
+      "sft_loss": 1.482043981552124,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 6.510991794442674,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": -0.10228048264980316,
+      "logits/rejected": 0.024892430752515793,
+      "logps/chosen": -1.4259611368179321,
+      "logps/rejected": -1.8754980564117432,
+      "loss": 0.4684,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4259611368179321,
+      "rewards/margins": 0.44953688979148865,
+      "rewards/rejected": -1.8754980564117432,
+      "sft_loss": 1.4861032962799072,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 7.365459217470868,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": -0.14073030650615692,
+      "logits/rejected": 0.06208374351263046,
+      "logps/chosen": -1.4707056283950806,
+      "logps/rejected": -1.9706439971923828,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4707056283950806,
+      "rewards/margins": 0.4999384880065918,
+      "rewards/rejected": -1.9706439971923828,
+      "sft_loss": 1.5462729930877686,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 6.119979287546052,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": -0.1096295490860939,
+      "logits/rejected": 0.11302530765533447,
+      "logps/chosen": -1.3502978086471558,
+      "logps/rejected": -1.9090697765350342,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3502978086471558,
+      "rewards/margins": 0.5587717890739441,
+      "rewards/rejected": -1.9090697765350342,
+      "sft_loss": 1.4756377935409546,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 8.220082955508166,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": -0.1352849304676056,
+      "logits/rejected": 0.0010965674882754683,
+      "logps/chosen": -1.4500648975372314,
+      "logps/rejected": -2.068101406097412,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4500648975372314,
+      "rewards/margins": 0.6180363893508911,
+      "rewards/rejected": -2.068101406097412,
+      "sft_loss": 1.5455083847045898,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 6.1082001469248,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": -0.15699729323387146,
+      "logits/rejected": 0.010936413891613483,
+      "logps/chosen": -1.3993462324142456,
+      "logps/rejected": -1.813114881515503,
+      "loss": 0.4811,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3993462324142456,
+      "rewards/margins": 0.4137686789035797,
+      "rewards/rejected": -1.813114881515503,
+      "sft_loss": 1.472827672958374,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 6.667631595880087,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": -0.10414779186248779,
+      "logits/rejected": 0.016749560832977295,
+      "logps/chosen": -1.547855257987976,
+      "logps/rejected": -2.1255764961242676,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.547855257987976,
+      "rewards/margins": 0.5777208209037781,
+      "rewards/rejected": -2.1255764961242676,
+      "sft_loss": 1.6323277950286865,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 8.921386278747955,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": -0.04205799847841263,
+      "logits/rejected": 0.12203924357891083,
+      "logps/chosen": -1.472687005996704,
+      "logps/rejected": -1.9264453649520874,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.472687005996704,
+      "rewards/margins": 0.4537584185600281,
+      "rewards/rejected": -1.9264453649520874,
+      "sft_loss": 1.5990089178085327,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 7.053570351514786,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": -0.1811997890472412,
+      "logits/rejected": 0.017085423693060875,
+      "logps/chosen": -1.586169719696045,
+      "logps/rejected": -2.2159647941589355,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.586169719696045,
+      "rewards/margins": 0.629794716835022,
+      "rewards/rejected": -2.2159647941589355,
+      "sft_loss": 1.6507060527801514,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 8.264947967152885,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": -0.09138262271881104,
+      "logits/rejected": 0.0653950497508049,
+      "logps/chosen": -1.5182015895843506,
+      "logps/rejected": -1.9837443828582764,
+      "loss": 0.4573,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5182015895843506,
+      "rewards/margins": 0.46554288268089294,
+      "rewards/rejected": -1.9837443828582764,
+      "sft_loss": 1.6368268728256226,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 6.413760604034631,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": -0.1785578727722168,
+      "logits/rejected": -0.09279811382293701,
+      "logps/chosen": -1.5066957473754883,
+      "logps/rejected": -1.8460357189178467,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5066957473754883,
+      "rewards/margins": 0.33934006094932556,
+      "rewards/rejected": -1.8460357189178467,
+      "sft_loss": 1.5868940353393555,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 9.701479183254484,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": -0.1419348418712616,
+      "logits/rejected": -0.08633621037006378,
+      "logps/chosen": -1.5398082733154297,
+      "logps/rejected": -2.1274075508117676,
+      "loss": 0.4736,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5398082733154297,
+      "rewards/margins": 0.5875992178916931,
+      "rewards/rejected": -2.1274075508117676,
+      "sft_loss": 1.6137285232543945,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 5.9921000870979215,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": -0.13307741284370422,
+      "logits/rejected": -0.019663050770759583,
+      "logps/chosen": -1.680048942565918,
+      "logps/rejected": -2.104114532470703,
+      "loss": 0.5418,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.680048942565918,
+      "rewards/margins": 0.4240652918815613,
+      "rewards/rejected": -2.104114532470703,
+      "sft_loss": 1.6894235610961914,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 13.089147452135542,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": -0.19363871216773987,
+      "logits/rejected": -0.031053265556693077,
+      "logps/chosen": -1.6516166925430298,
+      "logps/rejected": -2.150430679321289,
+      "loss": 0.5165,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6516166925430298,
+      "rewards/margins": 0.49881380796432495,
+      "rewards/rejected": -2.150430679321289,
+      "sft_loss": 1.6912845373153687,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 9.129385007733266,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": -0.07705952227115631,
+      "logits/rejected": -0.09640021622180939,
+      "logps/chosen": -1.5163434743881226,
+      "logps/rejected": -1.9805564880371094,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5163434743881226,
+      "rewards/margins": 0.4642130434513092,
+      "rewards/rejected": -1.9805564880371094,
+      "sft_loss": 1.6745860576629639,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 6.106860258760642,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": 0.02769961953163147,
+      "logits/rejected": 0.048429567366838455,
+      "logps/chosen": -1.3723499774932861,
+      "logps/rejected": -1.8381656408309937,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3723499774932861,
+      "rewards/margins": 0.46581578254699707,
+      "rewards/rejected": -1.8381656408309937,
+      "sft_loss": 1.431898593902588,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 10.749893862767616,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": -0.184439018368721,
+      "logits/rejected": -0.09069575369358063,
+      "logps/chosen": -1.565915822982788,
+      "logps/rejected": -2.0079472064971924,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.565915822982788,
+      "rewards/margins": 0.44203153252601624,
+      "rewards/rejected": -2.0079472064971924,
+      "sft_loss": 1.6745878458023071,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": 0.195302814245224,
+      "eval_logits/rejected": 0.29392915964126587,
+      "eval_logps/chosen": -1.5391197204589844,
+      "eval_logps/rejected": -2.0192654132843018,
+      "eval_loss": 0.49466973543167114,
+      "eval_rewards/accuracies": 0.6572700142860413,
+      "eval_rewards/chosen": -1.5391197204589844,
+      "eval_rewards/margins": 0.4801456332206726,
+      "eval_rewards/rejected": -2.0192654132843018,
+      "eval_runtime": 43.4371,
+      "eval_samples_per_second": 30.964,
+      "eval_sft_loss": 1.6154696941375732,
+      "eval_steps_per_second": 7.758,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 3.8318594608132135,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": -0.2765553593635559,
+      "logits/rejected": -0.11357270181179047,
+      "logps/chosen": -1.367326259613037,
+      "logps/rejected": -1.9252784252166748,
+      "loss": 0.426,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.367326259613037,
+      "rewards/margins": 0.5579522252082825,
+      "rewards/rejected": -1.9252784252166748,
+      "sft_loss": 1.488986849784851,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 9.834571664430086,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": -0.1395595520734787,
+      "logits/rejected": -0.08087310194969177,
+      "logps/chosen": -1.48549485206604,
+      "logps/rejected": -1.9511163234710693,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.48549485206604,
+      "rewards/margins": 0.46562162041664124,
+      "rewards/rejected": -1.9511163234710693,
+      "sft_loss": 1.6048133373260498,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 6.52049819582276,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": -0.07088275253772736,
+      "logits/rejected": 0.06389816105365753,
+      "logps/chosen": -1.4949347972869873,
+      "logps/rejected": -2.0141525268554688,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4949347972869873,
+      "rewards/margins": 0.5192179083824158,
+      "rewards/rejected": -2.0141525268554688,
+      "sft_loss": 1.5503588914871216,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 6.625902983275402,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": -0.1074337586760521,
+      "logits/rejected": 0.09632369130849838,
+      "logps/chosen": -1.5207237005233765,
+      "logps/rejected": -1.9671484231948853,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5207237005233765,
+      "rewards/margins": 0.44642454385757446,
+      "rewards/rejected": -1.9671484231948853,
+      "sft_loss": 1.576856017112732,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 7.208878180194657,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": -0.17939253151416779,
+      "logits/rejected": 0.022124072536826134,
+      "logps/chosen": -1.5701862573623657,
+      "logps/rejected": -2.147313117980957,
+      "loss": 0.4768,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5701862573623657,
+      "rewards/margins": 0.5771269798278809,
+      "rewards/rejected": -2.147313117980957,
+      "sft_loss": 1.6021369695663452,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 8.871421803027232,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": -0.11368022114038467,
+      "logits/rejected": 0.05845404416322708,
+      "logps/chosen": -1.5595228672027588,
+      "logps/rejected": -2.0032458305358887,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5595228672027588,
+      "rewards/margins": 0.44372302293777466,
+      "rewards/rejected": -2.0032458305358887,
+      "sft_loss": 1.6930491924285889,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 6.925181490630755,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": -0.11891146749258041,
+      "logits/rejected": 0.06260021775960922,
+      "logps/chosen": -1.5366853475570679,
+      "logps/rejected": -2.0646588802337646,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5366853475570679,
+      "rewards/margins": 0.527973473072052,
+      "rewards/rejected": -2.0646588802337646,
+      "sft_loss": 1.6199018955230713,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 7.008280760515188,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": -0.1227370947599411,
+      "logits/rejected": -0.018377363681793213,
+      "logps/chosen": -1.5639212131500244,
+      "logps/rejected": -1.9450569152832031,
+      "loss": 0.5215,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5639212131500244,
+      "rewards/margins": 0.3811357617378235,
+      "rewards/rejected": -1.9450569152832031,
+      "sft_loss": 1.6612507104873657,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 9.088827613891114,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": -0.08626069128513336,
+      "logits/rejected": 0.013733291998505592,
+      "logps/chosen": -1.5527054071426392,
+      "logps/rejected": -2.0260472297668457,
+      "loss": 0.5172,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5527054071426392,
+      "rewards/margins": 0.47334200143814087,
+      "rewards/rejected": -2.0260472297668457,
+      "sft_loss": 1.552793264389038,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 6.831388808036328,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": -0.1691809594631195,
+      "logits/rejected": -0.0750095397233963,
+      "logps/chosen": -1.5391981601715088,
+      "logps/rejected": -2.047201633453369,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5391981601715088,
+      "rewards/margins": 0.5080040693283081,
+      "rewards/rejected": -2.047201633453369,
+      "sft_loss": 1.60979425907135,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 7.175477010957574,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": -0.14673514664173126,
+      "logits/rejected": 0.0352972038090229,
+      "logps/chosen": -1.705213189125061,
+      "logps/rejected": -2.234771966934204,
+      "loss": 0.4749,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.705213189125061,
+      "rewards/margins": 0.5295583605766296,
+      "rewards/rejected": -2.234771966934204,
+      "sft_loss": 1.800531029701233,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 7.499618017199612,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": -0.21774613857269287,
+      "logits/rejected": -0.020393937826156616,
+      "logps/chosen": -1.4769175052642822,
+      "logps/rejected": -1.9039077758789062,
+      "loss": 0.4746,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4769175052642822,
+      "rewards/margins": 0.4269903600215912,
+      "rewards/rejected": -1.9039077758789062,
+      "sft_loss": 1.5506912469863892,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 6.416943109466104,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": -0.25113674998283386,
+      "logits/rejected": -0.10772594064474106,
+      "logps/chosen": -1.522179365158081,
+      "logps/rejected": -1.9736213684082031,
+      "loss": 0.4688,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.522179365158081,
+      "rewards/margins": 0.4514419436454773,
+      "rewards/rejected": -1.9736213684082031,
+      "sft_loss": 1.649560570716858,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 9.949473471115484,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": -0.13834629952907562,
+      "logits/rejected": -0.013131308369338512,
+      "logps/chosen": -1.5474927425384521,
+      "logps/rejected": -1.873211145401001,
+      "loss": 0.515,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5474927425384521,
+      "rewards/margins": 0.32571840286254883,
+      "rewards/rejected": -1.873211145401001,
+      "sft_loss": 1.581970453262329,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 7.615723711948691,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": -0.15234772861003876,
+      "logits/rejected": -0.038485340774059296,
+      "logps/chosen": -1.5660016536712646,
+      "logps/rejected": -2.0160014629364014,
+      "loss": 0.4617,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5660016536712646,
+      "rewards/margins": 0.4499998092651367,
+      "rewards/rejected": -2.0160014629364014,
+      "sft_loss": 1.5764434337615967,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 8.829175592430431,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": 0.002240369562059641,
+      "logits/rejected": 0.12692388892173767,
+      "logps/chosen": -1.4671860933303833,
+      "logps/rejected": -1.986998200416565,
+      "loss": 0.4817,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4671860933303833,
+      "rewards/margins": 0.5198121666908264,
+      "rewards/rejected": -1.986998200416565,
+      "sft_loss": 1.5353573560714722,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 8.050631038655702,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": -0.13125446438789368,
+      "logits/rejected": -0.0016770787769928575,
+      "logps/chosen": -1.6214059591293335,
+      "logps/rejected": -2.1163296699523926,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6214059591293335,
+      "rewards/margins": 0.4949234127998352,
+      "rewards/rejected": -2.1163296699523926,
+      "sft_loss": 1.7011600732803345,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 7.662816680327483,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": -0.202124685049057,
+      "logits/rejected": 0.001256814575754106,
+      "logps/chosen": -1.5663325786590576,
+      "logps/rejected": -2.1145505905151367,
+      "loss": 0.4669,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5663325786590576,
+      "rewards/margins": 0.5482179522514343,
+      "rewards/rejected": -2.1145505905151367,
+      "sft_loss": 1.6170275211334229,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 8.485306367150564,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": -0.17051918804645538,
+      "logits/rejected": -0.007482844404876232,
+      "logps/chosen": -1.4477908611297607,
+      "logps/rejected": -1.9668010473251343,
+      "loss": 0.4643,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4477908611297607,
+      "rewards/margins": 0.5190101861953735,
+      "rewards/rejected": -1.9668010473251343,
+      "sft_loss": 1.4976394176483154,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 5.4358669169930005,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": -0.0875912755727768,
+      "logits/rejected": 0.02966555580496788,
+      "logps/chosen": -1.5256271362304688,
+      "logps/rejected": -2.0319604873657227,
+      "loss": 0.4669,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5256271362304688,
+      "rewards/margins": 0.5063332319259644,
+      "rewards/rejected": -2.0319604873657227,
+      "sft_loss": 1.6018095016479492,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 6.746406325493352,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": -0.011767564341425896,
+      "logits/rejected": 0.01628190465271473,
+      "logps/chosen": -1.49928879737854,
+      "logps/rejected": -1.9732303619384766,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.49928879737854,
+      "rewards/margins": 0.4739416241645813,
+      "rewards/rejected": -1.9732303619384766,
+      "sft_loss": 1.584179162979126,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 6.6605557708847005,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": -0.08847250044345856,
+      "logits/rejected": 0.022497396916151047,
+      "logps/chosen": -1.51120126247406,
+      "logps/rejected": -1.901601791381836,
+      "loss": 0.4866,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.51120126247406,
+      "rewards/margins": 0.3904004693031311,
+      "rewards/rejected": -1.901601791381836,
+      "sft_loss": 1.6852781772613525,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 15.629320305461311,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": -0.03065609000623226,
+      "logits/rejected": 0.16193042695522308,
+      "logps/chosen": -1.5821765661239624,
+      "logps/rejected": -2.097710132598877,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5821765661239624,
+      "rewards/margins": 0.515533447265625,
+      "rewards/rejected": -2.097710132598877,
+      "sft_loss": 1.6894975900650024,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 8.017551981264566,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": -0.03545909374952316,
+      "logits/rejected": 0.10086387395858765,
+      "logps/chosen": -1.5606505870819092,
+      "logps/rejected": -2.073725461959839,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5606505870819092,
+      "rewards/margins": 0.5130751132965088,
+      "rewards/rejected": -2.073725461959839,
+      "sft_loss": 1.6506057977676392,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 7.474124977180021,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": -0.019739773124456406,
+      "logits/rejected": 0.05907121300697327,
+      "logps/chosen": -1.432644009590149,
+      "logps/rejected": -1.9527826309204102,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.432644009590149,
+      "rewards/margins": 0.520138680934906,
+      "rewards/rejected": -1.9527826309204102,
+      "sft_loss": 1.4820513725280762,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 7.125622116791099,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": -0.17671312391757965,
+      "logits/rejected": 0.02326101064682007,
+      "logps/chosen": -1.4796098470687866,
+      "logps/rejected": -2.236234664916992,
+      "loss": 0.431,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4796098470687866,
+      "rewards/margins": 0.7566248774528503,
+      "rewards/rejected": -2.236234664916992,
+      "sft_loss": 1.6504055261611938,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 8.608945768862661,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": -0.14483094215393066,
+      "logits/rejected": 0.013085213489830494,
+      "logps/chosen": -1.562831163406372,
+      "logps/rejected": -2.0342061519622803,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.562831163406372,
+      "rewards/margins": 0.47137507796287537,
+      "rewards/rejected": -2.0342061519622803,
+      "sft_loss": 1.6741364002227783,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 7.206925970451456,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": -0.1183919757604599,
+      "logits/rejected": 0.0802488699555397,
+      "logps/chosen": -1.523560881614685,
+      "logps/rejected": -2.1156466007232666,
+      "loss": 0.4657,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.523560881614685,
+      "rewards/margins": 0.5920857191085815,
+      "rewards/rejected": -2.1156466007232666,
+      "sft_loss": 1.641283631324768,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 10.318213241407362,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": -0.1034453734755516,
+      "logits/rejected": 0.05875151604413986,
+      "logps/chosen": -1.4753280878067017,
+      "logps/rejected": -1.8768761157989502,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4753280878067017,
+      "rewards/margins": 0.40154796838760376,
+      "rewards/rejected": -1.8768761157989502,
+      "sft_loss": 1.526484727859497,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 10.426324372194873,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": -0.008512385189533234,
+      "logits/rejected": 0.2818513810634613,
+      "logps/chosen": -1.4411323070526123,
+      "logps/rejected": -2.0921225547790527,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4411323070526123,
+      "rewards/margins": 0.6509900689125061,
+      "rewards/rejected": -2.0921225547790527,
+      "sft_loss": 1.5098687410354614,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 8.73690464393858,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": -0.020450295880436897,
+      "logits/rejected": 0.09912039339542389,
+      "logps/chosen": -1.5450700521469116,
+      "logps/rejected": -2.082106113433838,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5450700521469116,
+      "rewards/margins": 0.5370359420776367,
+      "rewards/rejected": -2.082106113433838,
+      "sft_loss": 1.6611696481704712,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 8.984068771825296,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": -0.12652283906936646,
+      "logits/rejected": 0.02124221995472908,
+      "logps/chosen": -1.5176342725753784,
+      "logps/rejected": -1.9752323627471924,
+      "loss": 0.4832,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5176342725753784,
+      "rewards/margins": 0.4575978219509125,
+      "rewards/rejected": -1.9752323627471924,
+      "sft_loss": 1.5860871076583862,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 6.047546625096609,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": -0.07259048521518707,
+      "logits/rejected": 0.034884002059698105,
+      "logps/chosen": -1.4993550777435303,
+      "logps/rejected": -1.9948831796646118,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4993550777435303,
+      "rewards/margins": 0.4955279231071472,
+      "rewards/rejected": -1.9948831796646118,
+      "sft_loss": 1.558639407157898,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 10.15818250535398,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": -0.10375696420669556,
+      "logits/rejected": 0.12088628858327866,
+      "logps/chosen": -1.610216498374939,
+      "logps/rejected": -2.1483049392700195,
+      "loss": 0.5139,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.610216498374939,
+      "rewards/margins": 0.5380884408950806,
+      "rewards/rejected": -2.1483049392700195,
+      "sft_loss": 1.7288093566894531,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 8.730149993908,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": -0.07916657626628876,
+      "logits/rejected": 0.09168480336666107,
+      "logps/chosen": -1.5467594861984253,
+      "logps/rejected": -2.1468772888183594,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5467594861984253,
+      "rewards/margins": 0.6001176238059998,
+      "rewards/rejected": -2.1468772888183594,
+      "sft_loss": 1.657918930053711,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 5.4880649780877,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": -0.15298187732696533,
+      "logits/rejected": 0.04283507913351059,
+      "logps/chosen": -1.6657731533050537,
+      "logps/rejected": -2.1596436500549316,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6657731533050537,
+      "rewards/margins": 0.49387043714523315,
+      "rewards/rejected": -2.1596436500549316,
+      "sft_loss": 1.7058131694793701,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 6.2781301998920735,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": -0.12671543657779694,
+      "logits/rejected": 0.09902218729257584,
+      "logps/chosen": -1.6928863525390625,
+      "logps/rejected": -2.257063388824463,
+      "loss": 0.5153,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6928863525390625,
+      "rewards/margins": 0.5641769766807556,
+      "rewards/rejected": -2.257063388824463,
+      "sft_loss": 1.6770423650741577,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 10.670619321805976,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": -0.10209260880947113,
+      "logits/rejected": 0.11269841343164444,
+      "logps/chosen": -1.517526388168335,
+      "logps/rejected": -2.184566020965576,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.517526388168335,
+      "rewards/margins": 0.667039692401886,
+      "rewards/rejected": -2.184566020965576,
+      "sft_loss": 1.5879290103912354,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 20.760880204872265,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": -0.08403249830007553,
+      "logits/rejected": 0.056710999459028244,
+      "logps/chosen": -1.6829345226287842,
+      "logps/rejected": -2.4216763973236084,
+      "loss": 0.467,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6829345226287842,
+      "rewards/margins": 0.7387418150901794,
+      "rewards/rejected": -2.4216763973236084,
+      "sft_loss": 1.8225390911102295,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 4.916358646100815,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": -0.10212262719869614,
+      "logits/rejected": 0.07346180081367493,
+      "logps/chosen": -1.5672862529754639,
+      "logps/rejected": -2.046978712081909,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5672862529754639,
+      "rewards/margins": 0.4796925485134125,
+      "rewards/rejected": -2.046978712081909,
+      "sft_loss": 1.574955701828003,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 11.006061549135756,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": -0.18173246085643768,
+      "logits/rejected": -0.011754069477319717,
+      "logps/chosen": -1.4685728549957275,
+      "logps/rejected": -1.8448057174682617,
+      "loss": 0.4863,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4685728549957275,
+      "rewards/margins": 0.3762328326702118,
+      "rewards/rejected": -1.8448057174682617,
+      "sft_loss": 1.5300872325897217,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 7.797550997747095,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": -0.158115416765213,
+      "logits/rejected": -0.010147708468139172,
+      "logps/chosen": -1.4935801029205322,
+      "logps/rejected": -1.8723909854888916,
+      "loss": 0.5091,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4935801029205322,
+      "rewards/margins": 0.37881091237068176,
+      "rewards/rejected": -1.8723909854888916,
+      "sft_loss": 1.5578104257583618,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 10.014579651548331,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": 0.06347858905792236,
+      "logits/rejected": 0.15640303492546082,
+      "logps/chosen": -1.4985144138336182,
+      "logps/rejected": -1.868246078491211,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4985144138336182,
+      "rewards/margins": 0.36973172426223755,
+      "rewards/rejected": -1.868246078491211,
+      "sft_loss": 1.5547329187393188,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 6.684457081770787,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": -0.13590192794799805,
+      "logits/rejected": 0.03512415289878845,
+      "logps/chosen": -1.483924388885498,
+      "logps/rejected": -1.9175758361816406,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.483924388885498,
+      "rewards/margins": 0.4336513578891754,
+      "rewards/rejected": -1.9175758361816406,
+      "sft_loss": 1.6244289875030518,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 6.5624449173763955,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": -0.08367796242237091,
+      "logits/rejected": 0.10011348873376846,
+      "logps/chosen": -1.4311835765838623,
+      "logps/rejected": -1.8682610988616943,
+      "loss": 0.4688,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4311835765838623,
+      "rewards/margins": 0.4370775818824768,
+      "rewards/rejected": -1.8682610988616943,
+      "sft_loss": 1.5716338157653809,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 7.1403128736198545,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": -0.06696267426013947,
+      "logits/rejected": 0.09421399235725403,
+      "logps/chosen": -1.5920861959457397,
+      "logps/rejected": -2.0236916542053223,
+      "loss": 0.5067,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5920861959457397,
+      "rewards/margins": 0.43160510063171387,
+      "rewards/rejected": -2.0236916542053223,
+      "sft_loss": 1.6656211614608765,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 8.899740715118822,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": 0.04497329145669937,
+      "logits/rejected": 0.1868835836648941,
+      "logps/chosen": -1.5424516201019287,
+      "logps/rejected": -1.8982648849487305,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5424516201019287,
+      "rewards/margins": 0.355813205242157,
+      "rewards/rejected": -1.8982648849487305,
+      "sft_loss": 1.6119035482406616,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 6.559493814754457,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": -0.0907437801361084,
+      "logits/rejected": -0.013587561435997486,
+      "logps/chosen": -1.4611659049987793,
+      "logps/rejected": -2.103156328201294,
+      "loss": 0.4392,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4611659049987793,
+      "rewards/margins": 0.6419903635978699,
+      "rewards/rejected": -2.103156328201294,
+      "sft_loss": 1.5914957523345947,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 14.776232738852986,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": -0.06494507193565369,
+      "logits/rejected": 0.05210378021001816,
+      "logps/chosen": -1.4210389852523804,
+      "logps/rejected": -1.815585732460022,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4210389852523804,
+      "rewards/margins": 0.3945468068122864,
+      "rewards/rejected": -1.815585732460022,
+      "sft_loss": 1.527193307876587,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 6.376406445542566,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": -0.1394258737564087,
+      "logits/rejected": 0.03964899107813835,
+      "logps/chosen": -1.5754724740982056,
+      "logps/rejected": -2.152021884918213,
+      "loss": 0.4591,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5754724740982056,
+      "rewards/margins": 0.5765495300292969,
+      "rewards/rejected": -2.152021884918213,
+      "sft_loss": 1.7064645290374756,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 7.174721499128706,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": -0.11442530155181885,
+      "logits/rejected": 0.05020016431808472,
+      "logps/chosen": -1.5213871002197266,
+      "logps/rejected": -2.025872230529785,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5213871002197266,
+      "rewards/margins": 0.5044850707054138,
+      "rewards/rejected": -2.025872230529785,
+      "sft_loss": 1.5640764236450195,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 5.183907088182196,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": -0.10387809574604034,
+      "logits/rejected": 0.10323204845190048,
+      "logps/chosen": -1.549683928489685,
+      "logps/rejected": -2.03615403175354,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.549683928489685,
+      "rewards/margins": 0.48646992444992065,
+      "rewards/rejected": -2.03615403175354,
+      "sft_loss": 1.6807247400283813,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 9.522880950039765,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": 9.657442205934785e-06,
+      "logits/rejected": 0.05333443731069565,
+      "logps/chosen": -1.5538901090621948,
+      "logps/rejected": -2.1399333477020264,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5538901090621948,
+      "rewards/margins": 0.5860432386398315,
+      "rewards/rejected": -2.1399333477020264,
+      "sft_loss": 1.6596462726593018,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 10.083793649113037,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": -0.1466716080904007,
+      "logits/rejected": -0.003812560345977545,
+      "logps/chosen": -1.662615180015564,
+      "logps/rejected": -2.175483226776123,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.662615180015564,
+      "rewards/margins": 0.51286780834198,
+      "rewards/rejected": -2.175483226776123,
+      "sft_loss": 1.7367607355117798,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 4.758845320619211,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": -0.1588786542415619,
+      "logits/rejected": -0.0798439010977745,
+      "logps/chosen": -1.4433513879776,
+      "logps/rejected": -1.896655797958374,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4433513879776,
+      "rewards/margins": 0.4533044695854187,
+      "rewards/rejected": -1.896655797958374,
+      "sft_loss": 1.4855616092681885,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 5.792721475907608,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": -0.2315061092376709,
+      "logits/rejected": -0.013959243893623352,
+      "logps/chosen": -1.4062265157699585,
+      "logps/rejected": -1.9291127920150757,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4062265157699585,
+      "rewards/margins": 0.5228861570358276,
+      "rewards/rejected": -1.9291127920150757,
+      "sft_loss": 1.5246238708496094,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 4.927542384265485,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": -0.01306633185595274,
+      "logits/rejected": 0.010603232309222221,
+      "logps/chosen": -1.4386698007583618,
+      "logps/rejected": -1.898354172706604,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4386698007583618,
+      "rewards/margins": 0.4596843719482422,
+      "rewards/rejected": -1.898354172706604,
+      "sft_loss": 1.5103987455368042,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 6.7826883374939415,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": 0.008967900648713112,
+      "logits/rejected": 0.10348598659038544,
+      "logps/chosen": -1.4943504333496094,
+      "logps/rejected": -1.8437782526016235,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4943504333496094,
+      "rewards/margins": 0.34942787885665894,
+      "rewards/rejected": -1.8437782526016235,
+      "sft_loss": 1.626360297203064,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 7.473960780833833,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": -0.05825355648994446,
+      "logits/rejected": 0.11429224908351898,
+      "logps/chosen": -1.5541985034942627,
+      "logps/rejected": -2.045668363571167,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5541985034942627,
+      "rewards/margins": 0.4914700388908386,
+      "rewards/rejected": -2.045668363571167,
+      "sft_loss": 1.628761649131775,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 7.821153511440808,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": -0.21912486851215363,
+      "logits/rejected": -0.06080486252903938,
+      "logps/chosen": -1.5123136043548584,
+      "logps/rejected": -2.0528147220611572,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5123136043548584,
+      "rewards/margins": 0.5405011773109436,
+      "rewards/rejected": -2.0528147220611572,
+      "sft_loss": 1.6570866107940674,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 8.65323848981444,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": -0.13585327565670013,
+      "logits/rejected": 0.0843258872628212,
+      "logps/chosen": -1.5429242849349976,
+      "logps/rejected": -1.9370723962783813,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5429242849349976,
+      "rewards/margins": 0.394148051738739,
+      "rewards/rejected": -1.9370723962783813,
+      "sft_loss": 1.6510931253433228,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 14.129405465063757,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": -0.05413060635328293,
+      "logits/rejected": 0.0819631814956665,
+      "logps/chosen": -1.5078423023223877,
+      "logps/rejected": -1.936669111251831,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5078423023223877,
+      "rewards/margins": 0.4288269579410553,
+      "rewards/rejected": -1.936669111251831,
+      "sft_loss": 1.4708707332611084,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 7.656275659256228,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": 0.04149172827601433,
+      "logits/rejected": 0.15403041243553162,
+      "logps/chosen": -1.557957410812378,
+      "logps/rejected": -2.077946424484253,
+      "loss": 0.4698,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.557957410812378,
+      "rewards/margins": 0.519989013671875,
+      "rewards/rejected": -2.077946424484253,
+      "sft_loss": 1.683276891708374,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 7.864028694833735,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": -0.12901760637760162,
+      "logits/rejected": 0.05095566436648369,
+      "logps/chosen": -1.592777132987976,
+      "logps/rejected": -2.107862949371338,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.592777132987976,
+      "rewards/margins": 0.5150859355926514,
+      "rewards/rejected": -2.107862949371338,
+      "sft_loss": 1.6193393468856812,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 6.254504666073017,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": -0.015704909339547157,
+      "logits/rejected": 0.12680071592330933,
+      "logps/chosen": -1.5865771770477295,
+      "logps/rejected": -2.06243896484375,
+      "loss": 0.5038,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5865771770477295,
+      "rewards/margins": 0.4758616089820862,
+      "rewards/rejected": -2.06243896484375,
+      "sft_loss": 1.5720789432525635,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 6.19753711179988,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": -0.025851398706436157,
+      "logits/rejected": 0.16065967082977295,
+      "logps/chosen": -1.61211359500885,
+      "logps/rejected": -2.105921506881714,
+      "loss": 0.473,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.61211359500885,
+      "rewards/margins": 0.4938080906867981,
+      "rewards/rejected": -2.105921506881714,
+      "sft_loss": 1.7118337154388428,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 7.017273961715333,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": 0.032778430730104446,
+      "logits/rejected": 0.11387556791305542,
+      "logps/chosen": -1.614831566810608,
+      "logps/rejected": -2.009108066558838,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.614831566810608,
+      "rewards/margins": 0.3942764401435852,
+      "rewards/rejected": -2.009108066558838,
+      "sft_loss": 1.5776565074920654,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 5.893771627132026,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": -0.25968000292778015,
+      "logits/rejected": -0.08750112354755402,
+      "logps/chosen": -1.424504280090332,
+      "logps/rejected": -1.8231842517852783,
+      "loss": 0.484,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.424504280090332,
+      "rewards/margins": 0.3986798822879791,
+      "rewards/rejected": -1.8231842517852783,
+      "sft_loss": 1.5326725244522095,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 6.211170635152287,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": -0.06294619292020798,
+      "logits/rejected": 0.10353779792785645,
+      "logps/chosen": -1.4389173984527588,
+      "logps/rejected": -1.9819748401641846,
+      "loss": 0.4727,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4389173984527588,
+      "rewards/margins": 0.543057382106781,
+      "rewards/rejected": -1.9819748401641846,
+      "sft_loss": 1.5338695049285889,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 8.202664843545957,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": -0.21335256099700928,
+      "logits/rejected": -0.07415179908275604,
+      "logps/chosen": -1.5558087825775146,
+      "logps/rejected": -2.019449472427368,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5558087825775146,
+      "rewards/margins": 0.4636405110359192,
+      "rewards/rejected": -2.019449472427368,
+      "sft_loss": 1.5690850019454956,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 7.555399006864592,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": -0.21016494929790497,
+      "logits/rejected": -0.07241862267255783,
+      "logps/chosen": -1.3800415992736816,
+      "logps/rejected": -1.8963416814804077,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3800415992736816,
+      "rewards/margins": 0.5163003206253052,
+      "rewards/rejected": -1.8963416814804077,
+      "sft_loss": 1.4642889499664307,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 8.021356820522469,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": -0.0793885663151741,
+      "logits/rejected": -0.0631255954504013,
+      "logps/chosen": -1.495445966720581,
+      "logps/rejected": -2.0048792362213135,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.495445966720581,
+      "rewards/margins": 0.5094331502914429,
+      "rewards/rejected": -2.0048792362213135,
+      "sft_loss": 1.6442674398422241,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 5.884767247203592,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": -0.21232256293296814,
+      "logits/rejected": -0.09611859917640686,
+      "logps/chosen": -1.5573443174362183,
+      "logps/rejected": -1.9870105981826782,
+      "loss": 0.5023,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5573443174362183,
+      "rewards/margins": 0.429666131734848,
+      "rewards/rejected": -1.9870105981826782,
+      "sft_loss": 1.6721751689910889,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 6.508656695974505,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": -0.19365619122982025,
+      "logits/rejected": -0.09000679850578308,
+      "logps/chosen": -1.5363733768463135,
+      "logps/rejected": -2.0378050804138184,
+      "loss": 0.487,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5363733768463135,
+      "rewards/margins": 0.5014316439628601,
+      "rewards/rejected": -2.0378050804138184,
+      "sft_loss": 1.603652000427246,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 15.440592748257846,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": -0.22241048514842987,
+      "logits/rejected": -0.10200734436511993,
+      "logps/chosen": -1.5882127285003662,
+      "logps/rejected": -1.9402246475219727,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5882127285003662,
+      "rewards/margins": 0.35201185941696167,
+      "rewards/rejected": -1.9402246475219727,
+      "sft_loss": 1.6542675495147705,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 6.565886157189213,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": -0.06484415382146835,
+      "logits/rejected": -0.006489972118288279,
+      "logps/chosen": -1.5401251316070557,
+      "logps/rejected": -2.1260883808135986,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5401251316070557,
+      "rewards/margins": 0.5859628915786743,
+      "rewards/rejected": -2.1260883808135986,
+      "sft_loss": 1.5851103067398071,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 6.891541955713481,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": -0.09114833176136017,
+      "logits/rejected": -0.05542738363146782,
+      "logps/chosen": -1.5016356706619263,
+      "logps/rejected": -1.8026351928710938,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5016356706619263,
+      "rewards/margins": 0.30099934339523315,
+      "rewards/rejected": -1.8026351928710938,
+      "sft_loss": 1.6295297145843506,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 6.923994823932767,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": -0.19895055890083313,
+      "logits/rejected": -0.01881379820406437,
+      "logps/chosen": -1.475109338760376,
+      "logps/rejected": -2.0406415462493896,
+      "loss": 0.4467,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.475109338760376,
+      "rewards/margins": 0.5655320882797241,
+      "rewards/rejected": -2.0406415462493896,
+      "sft_loss": 1.5153456926345825,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 7.188624836614871,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": -0.2148740589618683,
+      "logits/rejected": -0.09418477863073349,
+      "logps/chosen": -1.4688165187835693,
+      "logps/rejected": -1.9643265008926392,
+      "loss": 0.4726,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4688165187835693,
+      "rewards/margins": 0.4955100417137146,
+      "rewards/rejected": -1.9643265008926392,
+      "sft_loss": 1.48080575466156,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 8.296804746944462,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": -0.08745540678501129,
+      "logits/rejected": 0.06984991580247879,
+      "logps/chosen": -1.5411351919174194,
+      "logps/rejected": -2.0730738639831543,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5411351919174194,
+      "rewards/margins": 0.531938910484314,
+      "rewards/rejected": -2.0730738639831543,
+      "sft_loss": 1.5940454006195068,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": 0.21331138908863068,
+      "eval_logits/rejected": 0.3130818009376526,
+      "eval_logps/chosen": -1.5037399530410767,
+      "eval_logps/rejected": -1.9595329761505127,
+      "eval_loss": 0.49355337023735046,
+      "eval_rewards/accuracies": 0.6483679413795471,
+      "eval_rewards/chosen": -1.5037399530410767,
+      "eval_rewards/margins": 0.455793172121048,
+      "eval_rewards/rejected": -1.9595329761505127,
+      "eval_runtime": 43.4212,
+      "eval_samples_per_second": 30.976,
+      "eval_sft_loss": 1.575928807258606,
+      "eval_steps_per_second": 7.761,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 7.354230571529315,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": -0.2326853722333908,
+      "logits/rejected": -0.09422379732131958,
+      "logps/chosen": -1.472592830657959,
+      "logps/rejected": -1.9556305408477783,
+      "loss": 0.4852,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.472592830657959,
+      "rewards/margins": 0.4830375611782074,
+      "rewards/rejected": -1.9556305408477783,
+      "sft_loss": 1.4990339279174805,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 8.17477043186777,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": -0.2231977880001068,
+      "logits/rejected": -0.11281216144561768,
+      "logps/chosen": -1.4896111488342285,
+      "logps/rejected": -1.9424409866333008,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4896111488342285,
+      "rewards/margins": 0.4528297781944275,
+      "rewards/rejected": -1.9424409866333008,
+      "sft_loss": 1.561738133430481,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 7.828530935557396,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": -0.18882520496845245,
+      "logits/rejected": -0.049553271383047104,
+      "logps/chosen": -1.467195749282837,
+      "logps/rejected": -1.9305130243301392,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.467195749282837,
+      "rewards/margins": 0.4633170962333679,
+      "rewards/rejected": -1.9305130243301392,
+      "sft_loss": 1.5044894218444824,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 6.379942477646264,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": -0.21386948227882385,
+      "logits/rejected": 0.03034193255007267,
+      "logps/chosen": -1.6358089447021484,
+      "logps/rejected": -2.112433910369873,
+      "loss": 0.5479,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6358089447021484,
+      "rewards/margins": 0.4766252636909485,
+      "rewards/rejected": -2.112433910369873,
+      "sft_loss": 1.6175771951675415,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 7.627844508547667,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": -0.1952795684337616,
+      "logits/rejected": -0.056179117411375046,
+      "logps/chosen": -1.3949609994888306,
+      "logps/rejected": -1.951027274131775,
+      "loss": 0.446,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3949609994888306,
+      "rewards/margins": 0.5560663342475891,
+      "rewards/rejected": -1.951027274131775,
+      "sft_loss": 1.4932529926300049,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 8.166108560103117,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": -0.20898957550525665,
+      "logits/rejected": -0.03555748611688614,
+      "logps/chosen": -1.4980252981185913,
+      "logps/rejected": -2.201258420944214,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4980252981185913,
+      "rewards/margins": 0.7032328844070435,
+      "rewards/rejected": -2.201258420944214,
+      "sft_loss": 1.5434107780456543,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 6.480679064846322,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": -0.18776097893714905,
+      "logits/rejected": -0.10159311443567276,
+      "logps/chosen": -1.5042344331741333,
+      "logps/rejected": -2.028208017349243,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5042344331741333,
+      "rewards/margins": 0.5239735841751099,
+      "rewards/rejected": -2.028208017349243,
+      "sft_loss": 1.5815365314483643,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 7.34266127543648,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": -0.10263445228338242,
+      "logits/rejected": 0.02675381675362587,
+      "logps/chosen": -1.4704691171646118,
+      "logps/rejected": -2.0698206424713135,
+      "loss": 0.4565,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4704691171646118,
+      "rewards/margins": 0.5993512272834778,
+      "rewards/rejected": -2.0698206424713135,
+      "sft_loss": 1.4575080871582031,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 11.529074535047988,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": -0.2872398793697357,
+      "logits/rejected": -0.128311425447464,
+      "logps/chosen": -1.5303713083267212,
+      "logps/rejected": -2.069119691848755,
+      "loss": 0.482,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5303713083267212,
+      "rewards/margins": 0.5387482643127441,
+      "rewards/rejected": -2.069119691848755,
+      "sft_loss": 1.528638243675232,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 6.229660340308704,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": -0.08383894711732864,
+      "logits/rejected": 0.07514307647943497,
+      "logps/chosen": -1.5344181060791016,
+      "logps/rejected": -2.0951080322265625,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5344181060791016,
+      "rewards/margins": 0.5606899857521057,
+      "rewards/rejected": -2.0951080322265625,
+      "sft_loss": 1.594663381576538,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 5.422422697510295,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": -0.10217050462961197,
+      "logits/rejected": -0.02202165499329567,
+      "logps/chosen": -1.5932085514068604,
+      "logps/rejected": -1.9968818426132202,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5932085514068604,
+      "rewards/margins": 0.4036734104156494,
+      "rewards/rejected": -1.9968818426132202,
+      "sft_loss": 1.6095945835113525,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 8.694436413759197,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": -0.24071171879768372,
+      "logits/rejected": -0.08918756991624832,
+      "logps/chosen": -1.5446043014526367,
+      "logps/rejected": -2.0271828174591064,
+      "loss": 0.4833,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5446043014526367,
+      "rewards/margins": 0.4825782775878906,
+      "rewards/rejected": -2.0271828174591064,
+      "sft_loss": 1.616150140762329,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 9.264782264435171,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": -0.12294987589120865,
+      "logits/rejected": -0.005746960639953613,
+      "logps/chosen": -1.4709341526031494,
+      "logps/rejected": -2.0256457328796387,
+      "loss": 0.4702,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4709341526031494,
+      "rewards/margins": 0.5547112822532654,
+      "rewards/rejected": -2.0256457328796387,
+      "sft_loss": 1.5738205909729004,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 9.243662501602794,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": -0.13053932785987854,
+      "logits/rejected": 0.021624181419610977,
+      "logps/chosen": -1.4685636758804321,
+      "logps/rejected": -2.024125337600708,
+      "loss": 0.4577,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4685636758804321,
+      "rewards/margins": 0.5555616617202759,
+      "rewards/rejected": -2.024125337600708,
+      "sft_loss": 1.533262848854065,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 6.070545992868571,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": -0.2038416564464569,
+      "logits/rejected": 0.12645886838436127,
+      "logps/chosen": -1.5461547374725342,
+      "logps/rejected": -2.1197752952575684,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5461547374725342,
+      "rewards/margins": 0.5736207962036133,
+      "rewards/rejected": -2.1197752952575684,
+      "sft_loss": 1.6438357830047607,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 9.534324648344255,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": -0.16504624485969543,
+      "logits/rejected": -0.058779411017894745,
+      "logps/chosen": -1.3824862241744995,
+      "logps/rejected": -1.8809837102890015,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3824862241744995,
+      "rewards/margins": 0.4984973073005676,
+      "rewards/rejected": -1.8809837102890015,
+      "sft_loss": 1.5216165781021118,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 6.194222891691346,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": -0.19524803757667542,
+      "logits/rejected": 0.04105467349290848,
+      "logps/chosen": -1.4834949970245361,
+      "logps/rejected": -2.0216991901397705,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4834949970245361,
+      "rewards/margins": 0.5382041931152344,
+      "rewards/rejected": -2.0216991901397705,
+      "sft_loss": 1.5563383102416992,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 8.985288741061424,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": -0.11997727304697037,
+      "logits/rejected": 0.0095209376886487,
+      "logps/chosen": -1.4815948009490967,
+      "logps/rejected": -1.9493820667266846,
+      "loss": 0.482,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4815948009490967,
+      "rewards/margins": 0.4677874445915222,
+      "rewards/rejected": -1.9493820667266846,
+      "sft_loss": 1.5820047855377197,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 6.636082849373608,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": -0.159527987241745,
+      "logits/rejected": -0.08030469715595245,
+      "logps/chosen": -1.659623146057129,
+      "logps/rejected": -2.1283786296844482,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.659623146057129,
+      "rewards/margins": 0.4687553942203522,
+      "rewards/rejected": -2.1283786296844482,
+      "sft_loss": 1.684664011001587,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 9.20108436395257,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": -0.28877630829811096,
+      "logits/rejected": -0.13612669706344604,
+      "logps/chosen": -1.6840623617172241,
+      "logps/rejected": -2.2746193408966064,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6840623617172241,
+      "rewards/margins": 0.5905566811561584,
+      "rewards/rejected": -2.2746193408966064,
+      "sft_loss": 1.7702968120574951,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 9.077056353232631,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": -0.24776725471019745,
+      "logits/rejected": -0.07644428312778473,
+      "logps/chosen": -1.6167583465576172,
+      "logps/rejected": -2.170414447784424,
+      "loss": 0.4828,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6167583465576172,
+      "rewards/margins": 0.553656280040741,
+      "rewards/rejected": -2.170414447784424,
+      "sft_loss": 1.6717586517333984,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 6.04205711769031,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": -0.03785474970936775,
+      "logits/rejected": 0.021119682118296623,
+      "logps/chosen": -1.5902318954467773,
+      "logps/rejected": -2.062140941619873,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5902318954467773,
+      "rewards/margins": 0.47190895676612854,
+      "rewards/rejected": -2.062140941619873,
+      "sft_loss": 1.6707704067230225,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 11.243449946581617,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": -0.2524152398109436,
+      "logits/rejected": -0.07547532767057419,
+      "logps/chosen": -1.5760393142700195,
+      "logps/rejected": -2.067971706390381,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5760393142700195,
+      "rewards/margins": 0.49193257093429565,
+      "rewards/rejected": -2.067971706390381,
+      "sft_loss": 1.6471372842788696,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 6.114760882832403,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": -0.13797271251678467,
+      "logits/rejected": 0.08544354140758514,
+      "logps/chosen": -1.5435459613800049,
+      "logps/rejected": -2.251192569732666,
+      "loss": 0.4458,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5435459613800049,
+      "rewards/margins": 0.7076464891433716,
+      "rewards/rejected": -2.251192569732666,
+      "sft_loss": 1.643958330154419,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 7.098971135859354,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": -0.21761508285999298,
+      "logits/rejected": -0.0269846860319376,
+      "logps/chosen": -1.5963890552520752,
+      "logps/rejected": -2.1412088871002197,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5963890552520752,
+      "rewards/margins": 0.5448198318481445,
+      "rewards/rejected": -2.1412088871002197,
+      "sft_loss": 1.7654186487197876,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 7.508412069749556,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": -0.09692569822072983,
+      "logits/rejected": 0.03345852345228195,
+      "logps/chosen": -1.4379321336746216,
+      "logps/rejected": -2.0407073497772217,
+      "loss": 0.4442,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4379321336746216,
+      "rewards/margins": 0.6027752757072449,
+      "rewards/rejected": -2.0407073497772217,
+      "sft_loss": 1.5631394386291504,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 23.373034792821425,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": -0.1263446807861328,
+      "logits/rejected": 0.04607892036437988,
+      "logps/chosen": -1.598233699798584,
+      "logps/rejected": -2.06453275680542,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.598233699798584,
+      "rewards/margins": 0.4662991464138031,
+      "rewards/rejected": -2.06453275680542,
+      "sft_loss": 1.600513219833374,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 6.915232794749861,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": -0.09540759027004242,
+      "logits/rejected": 0.08306506276130676,
+      "logps/chosen": -1.4474116563796997,
+      "logps/rejected": -1.98406982421875,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4474116563796997,
+      "rewards/margins": 0.5366581082344055,
+      "rewards/rejected": -1.98406982421875,
+      "sft_loss": 1.5605659484863281,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 6.827983381630936,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": -0.12237314134836197,
+      "logits/rejected": 0.07193153351545334,
+      "logps/chosen": -1.3919322490692139,
+      "logps/rejected": -1.8691747188568115,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3919322490692139,
+      "rewards/margins": 0.4772423803806305,
+      "rewards/rejected": -1.8691747188568115,
+      "sft_loss": 1.5088390111923218,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 8.547269408446702,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": -0.28706783056259155,
+      "logits/rejected": -0.12526081502437592,
+      "logps/chosen": -1.5753610134124756,
+      "logps/rejected": -1.9887001514434814,
+      "loss": 0.496,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5753610134124756,
+      "rewards/margins": 0.4133389890193939,
+      "rewards/rejected": -1.9887001514434814,
+      "sft_loss": 1.6392730474472046,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 6.253426926775852,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": -0.02050682343542576,
+      "logits/rejected": 0.031059980392456055,
+      "logps/chosen": -1.6005933284759521,
+      "logps/rejected": -1.961517095565796,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.6005933284759521,
+      "rewards/margins": 0.3609238266944885,
+      "rewards/rejected": -1.961517095565796,
+      "sft_loss": 1.6103566884994507,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 7.896506850980646,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": -0.1889674961566925,
+      "logits/rejected": 0.057184554636478424,
+      "logps/chosen": -1.4576613903045654,
+      "logps/rejected": -1.9298509359359741,
+      "loss": 0.4801,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4576613903045654,
+      "rewards/margins": 0.4721898138523102,
+      "rewards/rejected": -1.9298509359359741,
+      "sft_loss": 1.5358812808990479,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 5.739958509027284,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": -0.18853381276130676,
+      "logits/rejected": -0.010739481076598167,
+      "logps/chosen": -1.5681513547897339,
+      "logps/rejected": -2.059718608856201,
+      "loss": 0.4769,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5681513547897339,
+      "rewards/margins": 0.4915672242641449,
+      "rewards/rejected": -2.059718608856201,
+      "sft_loss": 1.6615225076675415,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 9.963203520040665,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": -0.19930417835712433,
+      "logits/rejected": -0.023639291524887085,
+      "logps/chosen": -1.5441948175430298,
+      "logps/rejected": -1.9339659214019775,
+      "loss": 0.5351,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5441948175430298,
+      "rewards/margins": 0.389771044254303,
+      "rewards/rejected": -1.9339659214019775,
+      "sft_loss": 1.636212706565857,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 10.36044116532147,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": -0.03929683566093445,
+      "logits/rejected": 0.030060648918151855,
+      "logps/chosen": -1.638843297958374,
+      "logps/rejected": -2.066988468170166,
+      "loss": 0.5512,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.638843297958374,
+      "rewards/margins": 0.42814522981643677,
+      "rewards/rejected": -2.066988468170166,
+      "sft_loss": 1.6774746179580688,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 5.500783193468918,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": -0.23853763937950134,
+      "logits/rejected": -0.06972865760326385,
+      "logps/chosen": -1.5138180255889893,
+      "logps/rejected": -2.024914026260376,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5138180255889893,
+      "rewards/margins": 0.5110958218574524,
+      "rewards/rejected": -2.024914026260376,
+      "sft_loss": 1.5387523174285889,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 5.7217239160253,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": -0.21125145256519318,
+      "logits/rejected": -0.17624393105506897,
+      "logps/chosen": -1.6046793460845947,
+      "logps/rejected": -1.986384630203247,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6046793460845947,
+      "rewards/margins": 0.3817053735256195,
+      "rewards/rejected": -1.986384630203247,
+      "sft_loss": 1.6781991720199585,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 9.763318573444817,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": -0.15552251040935516,
+      "logits/rejected": -0.016761211678385735,
+      "logps/chosen": -1.5631591081619263,
+      "logps/rejected": -1.9353306293487549,
+      "loss": 0.5276,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5631591081619263,
+      "rewards/margins": 0.3721713423728943,
+      "rewards/rejected": -1.9353306293487549,
+      "sft_loss": 1.6414425373077393,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 8.200539388456589,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": -0.18579599261283875,
+      "logits/rejected": -0.06101926043629646,
+      "logps/chosen": -1.429839849472046,
+      "logps/rejected": -1.963600754737854,
+      "loss": 0.4626,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.429839849472046,
+      "rewards/margins": 0.5337609052658081,
+      "rewards/rejected": -1.963600754737854,
+      "sft_loss": 1.4940623044967651,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 8.778522639839275,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": -0.15336444973945618,
+      "logits/rejected": -0.025536376982927322,
+      "logps/chosen": -1.4731427431106567,
+      "logps/rejected": -1.906964898109436,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4731427431106567,
+      "rewards/margins": 0.4338221549987793,
+      "rewards/rejected": -1.906964898109436,
+      "sft_loss": 1.5429713726043701,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 6.1224910256530745,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": -0.15219996869564056,
+      "logits/rejected": 0.021372251212596893,
+      "logps/chosen": -1.4793925285339355,
+      "logps/rejected": -1.9215284585952759,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4793925285339355,
+      "rewards/margins": 0.44213590025901794,
+      "rewards/rejected": -1.9215284585952759,
+      "sft_loss": 1.5823194980621338,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 11.156015775060384,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": -0.16902689635753632,
+      "logits/rejected": 0.09002505242824554,
+      "logps/chosen": -1.6346511840820312,
+      "logps/rejected": -2.040658473968506,
+      "loss": 0.5144,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6346511840820312,
+      "rewards/margins": 0.4060073792934418,
+      "rewards/rejected": -2.040658473968506,
+      "sft_loss": 1.6519008874893188,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 11.681531329961869,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": -0.1769445389509201,
+      "logits/rejected": -0.048740632832050323,
+      "logps/chosen": -1.5493731498718262,
+      "logps/rejected": -2.083101272583008,
+      "loss": 0.49,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5493731498718262,
+      "rewards/margins": 0.5337280631065369,
+      "rewards/rejected": -2.083101272583008,
+      "sft_loss": 1.6823348999023438,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 12.745222763832293,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": -0.1154409795999527,
+      "logits/rejected": 0.015516482293605804,
+      "logps/chosen": -1.614915132522583,
+      "logps/rejected": -2.193376302719116,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.614915132522583,
+      "rewards/margins": 0.5784613490104675,
+      "rewards/rejected": -2.193376302719116,
+      "sft_loss": 1.7101062536239624,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 5.527779575795557,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": -0.1938527524471283,
+      "logits/rejected": -0.1387048065662384,
+      "logps/chosen": -1.4739712476730347,
+      "logps/rejected": -1.9787203073501587,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4739712476730347,
+      "rewards/margins": 0.5047491192817688,
+      "rewards/rejected": -1.9787203073501587,
+      "sft_loss": 1.5416609048843384,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 6.6023513847062,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": -0.19061097502708435,
+      "logits/rejected": -0.047849975526332855,
+      "logps/chosen": -1.5689013004302979,
+      "logps/rejected": -2.236057758331299,
+      "loss": 0.4749,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5689013004302979,
+      "rewards/margins": 0.6671566367149353,
+      "rewards/rejected": -2.236057758331299,
+      "sft_loss": 1.6300235986709595,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 7.943540586218268,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": -0.18472620844841003,
+      "logits/rejected": -0.07400848716497421,
+      "logps/chosen": -1.4624192714691162,
+      "logps/rejected": -2.032019853591919,
+      "loss": 0.436,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4624192714691162,
+      "rewards/margins": 0.569600522518158,
+      "rewards/rejected": -2.032019853591919,
+      "sft_loss": 1.5836392641067505,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 8.677362616663258,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": -0.21190118789672852,
+      "logits/rejected": -0.13000845909118652,
+      "logps/chosen": -1.41072678565979,
+      "logps/rejected": -2.0230135917663574,
+      "loss": 0.4718,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.41072678565979,
+      "rewards/margins": 0.6122865676879883,
+      "rewards/rejected": -2.0230135917663574,
+      "sft_loss": 1.473109245300293,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 5.332106728284266,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": -0.21277959644794464,
+      "logits/rejected": 0.03557942062616348,
+      "logps/chosen": -1.5381152629852295,
+      "logps/rejected": -2.060868740081787,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5381152629852295,
+      "rewards/margins": 0.5227535367012024,
+      "rewards/rejected": -2.060868740081787,
+      "sft_loss": 1.616860032081604,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 8.743005708525434,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": -0.31704509258270264,
+      "logits/rejected": -0.11927709728479385,
+      "logps/chosen": -1.5102241039276123,
+      "logps/rejected": -2.0983102321624756,
+      "loss": 0.465,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5102241039276123,
+      "rewards/margins": 0.5880860090255737,
+      "rewards/rejected": -2.0983102321624756,
+      "sft_loss": 1.6415538787841797,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 6.495625379078563,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": -0.15546521544456482,
+      "logits/rejected": 0.04172942787408829,
+      "logps/chosen": -1.7578779458999634,
+      "logps/rejected": -2.3052031993865967,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7578779458999634,
+      "rewards/margins": 0.5473250150680542,
+      "rewards/rejected": -2.3052031993865967,
+      "sft_loss": 1.8285331726074219,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 9.959675323722992,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": -0.04424437880516052,
+      "logits/rejected": 0.09680449217557907,
+      "logps/chosen": -1.5204256772994995,
+      "logps/rejected": -2.0713820457458496,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5204256772994995,
+      "rewards/margins": 0.5509562492370605,
+      "rewards/rejected": -2.0713820457458496,
+      "sft_loss": 1.589459776878357,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 6.20212722185588,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": -0.20593221485614777,
+      "logits/rejected": -0.0690714567899704,
+      "logps/chosen": -1.4725421667099,
+      "logps/rejected": -1.945373773574829,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4725421667099,
+      "rewards/margins": 0.47283148765563965,
+      "rewards/rejected": -1.945373773574829,
+      "sft_loss": 1.5715610980987549,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 8.217769659561885,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": -0.23379914462566376,
+      "logits/rejected": -0.13025447726249695,
+      "logps/chosen": -1.6423708200454712,
+      "logps/rejected": -2.1803269386291504,
+      "loss": 0.4927,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6423708200454712,
+      "rewards/margins": 0.5379562973976135,
+      "rewards/rejected": -2.1803269386291504,
+      "sft_loss": 1.752219796180725,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 12.838511306887499,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": -0.22167137265205383,
+      "logits/rejected": -0.14777874946594238,
+      "logps/chosen": -1.5322400331497192,
+      "logps/rejected": -1.9573066234588623,
+      "loss": 0.4892,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5322400331497192,
+      "rewards/margins": 0.4250665605068207,
+      "rewards/rejected": -1.9573066234588623,
+      "sft_loss": 1.5717178583145142,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 5.921295803022303,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": -0.06847834587097168,
+      "logits/rejected": -0.023401662707328796,
+      "logps/chosen": -1.6123988628387451,
+      "logps/rejected": -2.10270619392395,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6123988628387451,
+      "rewards/margins": 0.4903070330619812,
+      "rewards/rejected": -2.10270619392395,
+      "sft_loss": 1.6930005550384521,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 5.80656323015529,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": -0.11366887390613556,
+      "logits/rejected": 0.013406882993876934,
+      "logps/chosen": -1.5091646909713745,
+      "logps/rejected": -1.9836050271987915,
+      "loss": 0.4801,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5091646909713745,
+      "rewards/margins": 0.4744402766227722,
+      "rewards/rejected": -1.9836050271987915,
+      "sft_loss": 1.6238653659820557,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 8.682702224715547,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": -0.11430926620960236,
+      "logits/rejected": 0.03476153686642647,
+      "logps/chosen": -1.548940658569336,
+      "logps/rejected": -1.9343265295028687,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.548940658569336,
+      "rewards/margins": 0.3853859603404999,
+      "rewards/rejected": -1.9343265295028687,
+      "sft_loss": 1.6631542444229126,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 8.461362933358332,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": -0.2042984962463379,
+      "logits/rejected": -0.011971579864621162,
+      "logps/chosen": -1.618351697921753,
+      "logps/rejected": -2.0287108421325684,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.618351697921753,
+      "rewards/margins": 0.41035908460617065,
+      "rewards/rejected": -2.0287108421325684,
+      "sft_loss": 1.7515051364898682,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 5.96404845236303,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": -0.11476609855890274,
+      "logits/rejected": 0.05753784254193306,
+      "logps/chosen": -1.5358325242996216,
+      "logps/rejected": -2.1429457664489746,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5358325242996216,
+      "rewards/margins": 0.6071134805679321,
+      "rewards/rejected": -2.1429457664489746,
+      "sft_loss": 1.6601890325546265,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 10.451011548578464,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": -0.13792508840560913,
+      "logits/rejected": 0.020068520680069923,
+      "logps/chosen": -1.5533324480056763,
+      "logps/rejected": -2.105034351348877,
+      "loss": 0.47,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5533324480056763,
+      "rewards/margins": 0.5517022609710693,
+      "rewards/rejected": -2.105034351348877,
+      "sft_loss": 1.6532182693481445,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 10.590718385432426,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": -0.1606968343257904,
+      "logits/rejected": -0.0639537051320076,
+      "logps/chosen": -1.4895262718200684,
+      "logps/rejected": -2.011099100112915,
+      "loss": 0.4611,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4895262718200684,
+      "rewards/margins": 0.5215727090835571,
+      "rewards/rejected": -2.011099100112915,
+      "sft_loss": 1.5524523258209229,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 7.410572086527162,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": -0.17349793016910553,
+      "logits/rejected": 0.04328025132417679,
+      "logps/chosen": -1.5520845651626587,
+      "logps/rejected": -2.1199307441711426,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5520845651626587,
+      "rewards/margins": 0.5678460001945496,
+      "rewards/rejected": -2.1199307441711426,
+      "sft_loss": 1.5678256750106812,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 6.852317477346541,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": -0.22616633772850037,
+      "logits/rejected": -0.058461517095565796,
+      "logps/chosen": -1.5593931674957275,
+      "logps/rejected": -2.1259655952453613,
+      "loss": 0.4745,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5593931674957275,
+      "rewards/margins": 0.5665722489356995,
+      "rewards/rejected": -2.1259655952453613,
+      "sft_loss": 1.6664631366729736,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 6.478689285386118,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": -0.12069617211818695,
+      "logits/rejected": 0.09780598431825638,
+      "logps/chosen": -1.5308897495269775,
+      "logps/rejected": -2.0637001991271973,
+      "loss": 0.4709,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5308897495269775,
+      "rewards/margins": 0.5328103303909302,
+      "rewards/rejected": -2.0637001991271973,
+      "sft_loss": 1.6409518718719482,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 8.604893947210467,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": -0.10809512436389923,
+      "logits/rejected": -0.02255006693303585,
+      "logps/chosen": -1.5483064651489258,
+      "logps/rejected": -2.0952706336975098,
+      "loss": 0.466,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5483064651489258,
+      "rewards/margins": 0.5469641089439392,
+      "rewards/rejected": -2.0952706336975098,
+      "sft_loss": 1.6761009693145752,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 9.206865932072462,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": -0.23768194019794464,
+      "logits/rejected": -0.08524081856012344,
+      "logps/chosen": -1.4865976572036743,
+      "logps/rejected": -2.0263915061950684,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4865976572036743,
+      "rewards/margins": 0.5397937893867493,
+      "rewards/rejected": -2.0263915061950684,
+      "sft_loss": 1.6183054447174072,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 10.922365841022103,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": -0.2043650597333908,
+      "logits/rejected": 0.04252279922366142,
+      "logps/chosen": -1.570642113685608,
+      "logps/rejected": -1.981335997581482,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.570642113685608,
+      "rewards/margins": 0.4106937348842621,
+      "rewards/rejected": -1.981335997581482,
+      "sft_loss": 1.660813570022583,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 7.6979737502482655,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": -0.3521515727043152,
+      "logits/rejected": -0.04693521186709404,
+      "logps/chosen": -1.546839714050293,
+      "logps/rejected": -2.0670905113220215,
+      "loss": 0.4612,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.546839714050293,
+      "rewards/margins": 0.520250678062439,
+      "rewards/rejected": -2.0670905113220215,
+      "sft_loss": 1.6589651107788086,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 7.046326412078959,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": -0.12103285640478134,
+      "logits/rejected": -0.01766505278646946,
+      "logps/chosen": -1.55022394657135,
+      "logps/rejected": -1.9817107915878296,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.55022394657135,
+      "rewards/margins": 0.4314870834350586,
+      "rewards/rejected": -1.9817107915878296,
+      "sft_loss": 1.5364574193954468,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 7.880188076428199,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": -0.27796703577041626,
+      "logits/rejected": -0.11648659408092499,
+      "logps/chosen": -1.479952096939087,
+      "logps/rejected": -2.0038201808929443,
+      "loss": 0.49,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.479952096939087,
+      "rewards/margins": 0.5238681435585022,
+      "rewards/rejected": -2.0038201808929443,
+      "sft_loss": 1.5604270696640015,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 7.573845511505143,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": -0.16289106011390686,
+      "logits/rejected": -0.06578390300273895,
+      "logps/chosen": -1.5211942195892334,
+      "logps/rejected": -2.022411584854126,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5211942195892334,
+      "rewards/margins": 0.5012173056602478,
+      "rewards/rejected": -2.022411584854126,
+      "sft_loss": 1.5894739627838135,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 5.366824693587982,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": -0.16283658146858215,
+      "logits/rejected": -0.07833613455295563,
+      "logps/chosen": -1.460172414779663,
+      "logps/rejected": -2.1396899223327637,
+      "loss": 0.4387,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.460172414779663,
+      "rewards/margins": 0.6795173287391663,
+      "rewards/rejected": -2.1396899223327637,
+      "sft_loss": 1.607731819152832,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 7.529517971883579,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": -0.23355989158153534,
+      "logits/rejected": -0.07068635523319244,
+      "logps/chosen": -1.5029217004776,
+      "logps/rejected": -2.018176794052124,
+      "loss": 0.4548,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5029217004776,
+      "rewards/margins": 0.5152554512023926,
+      "rewards/rejected": -2.018176794052124,
+      "sft_loss": 1.5598933696746826,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 5.399445925758498,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": -0.10982157289981842,
+      "logits/rejected": 0.03126415237784386,
+      "logps/chosen": -1.5024158954620361,
+      "logps/rejected": -2.045030355453491,
+      "loss": 0.4849,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5024158954620361,
+      "rewards/margins": 0.5426144003868103,
+      "rewards/rejected": -2.045030355453491,
+      "sft_loss": 1.5932767391204834,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 5.710608786017775,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": -0.1709875911474228,
+      "logits/rejected": -0.10945823043584824,
+      "logps/chosen": -1.4984357357025146,
+      "logps/rejected": -2.104400634765625,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4984357357025146,
+      "rewards/margins": 0.6059646606445312,
+      "rewards/rejected": -2.104400634765625,
+      "sft_loss": 1.5858982801437378,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 7.429716993450721,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": -0.08576101809740067,
+      "logits/rejected": -0.017979338765144348,
+      "logps/chosen": -1.6544307470321655,
+      "logps/rejected": -2.1500606536865234,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6544307470321655,
+      "rewards/margins": 0.4956297278404236,
+      "rewards/rejected": -2.1500606536865234,
+      "sft_loss": 1.7742359638214111,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 7.303360436549522,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": -0.13903836905956268,
+      "logits/rejected": -0.05833768844604492,
+      "logps/chosen": -1.598507285118103,
+      "logps/rejected": -1.9958927631378174,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.598507285118103,
+      "rewards/margins": 0.3973854184150696,
+      "rewards/rejected": -1.9958927631378174,
+      "sft_loss": 1.6810901165008545,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 6.7050619799100195,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": -0.29747599363327026,
+      "logits/rejected": -0.07863298803567886,
+      "logps/chosen": -1.525634527206421,
+      "logps/rejected": -2.047628879547119,
+      "loss": 0.4734,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.525634527206421,
+      "rewards/margins": 0.521994411945343,
+      "rewards/rejected": -2.047628879547119,
+      "sft_loss": 1.496787667274475,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 7.263561975702709,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": -0.12187772989273071,
+      "logits/rejected": 0.06612186878919601,
+      "logps/chosen": -1.561301827430725,
+      "logps/rejected": -2.2543601989746094,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.561301827430725,
+      "rewards/margins": 0.6930583715438843,
+      "rewards/rejected": -2.2543601989746094,
+      "sft_loss": 1.6595875024795532,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": 0.20892609655857086,
+      "eval_logits/rejected": 0.31129610538482666,
+      "eval_logps/chosen": -1.564028024673462,
+      "eval_logps/rejected": -2.076958179473877,
+      "eval_loss": 0.4893736243247986,
+      "eval_rewards/accuracies": 0.6661720871925354,
+      "eval_rewards/chosen": -1.564028024673462,
+      "eval_rewards/margins": 0.5129303336143494,
+      "eval_rewards/rejected": -2.076958179473877,
+      "eval_runtime": 43.2822,
+      "eval_samples_per_second": 31.075,
+      "eval_sft_loss": 1.6467366218566895,
+      "eval_steps_per_second": 7.786,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 7.70326214265193,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": -0.21333487331867218,
+      "logits/rejected": -0.16703540086746216,
+      "logps/chosen": -1.5174494981765747,
+      "logps/rejected": -1.9064096212387085,
+      "loss": 0.5063,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5174494981765747,
+      "rewards/margins": 0.3889603018760681,
+      "rewards/rejected": -1.9064096212387085,
+      "sft_loss": 1.6115996837615967,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 6.484089558922941,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": -0.17045536637306213,
+      "logits/rejected": -0.04207264631986618,
+      "logps/chosen": -1.5239019393920898,
+      "logps/rejected": -2.0207948684692383,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5239019393920898,
+      "rewards/margins": 0.49689292907714844,
+      "rewards/rejected": -2.0207948684692383,
+      "sft_loss": 1.6819967031478882,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 6.901103706125059,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": -0.11860332638025284,
+      "logits/rejected": 0.0086726825684309,
+      "logps/chosen": -1.583103895187378,
+      "logps/rejected": -2.07374906539917,
+      "loss": 0.4924,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.583103895187378,
+      "rewards/margins": 0.4906453490257263,
+      "rewards/rejected": -2.07374906539917,
+      "sft_loss": 1.6645939350128174,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 11.67293328945311,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": -0.16560427844524384,
+      "logits/rejected": -0.08525630831718445,
+      "logps/chosen": -1.642525315284729,
+      "logps/rejected": -2.016528844833374,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.642525315284729,
+      "rewards/margins": 0.3740035593509674,
+      "rewards/rejected": -2.016528844833374,
+      "sft_loss": 1.735414743423462,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 7.779367874057267,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": -0.10910829156637192,
+      "logits/rejected": -0.07743573188781738,
+      "logps/chosen": -1.5290305614471436,
+      "logps/rejected": -1.9644886255264282,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5290305614471436,
+      "rewards/margins": 0.43545812368392944,
+      "rewards/rejected": -1.9644886255264282,
+      "sft_loss": 1.6208540201187134,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 13.173882267171926,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": -0.20582859218120575,
+      "logits/rejected": -0.1449136734008789,
+      "logps/chosen": -1.5608258247375488,
+      "logps/rejected": -1.9999399185180664,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5608258247375488,
+      "rewards/margins": 0.43911415338516235,
+      "rewards/rejected": -1.9999399185180664,
+      "sft_loss": 1.7072786092758179,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 7.197716852516967,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": -0.22931042313575745,
+      "logits/rejected": -0.13184943795204163,
+      "logps/chosen": -1.4989603757858276,
+      "logps/rejected": -1.9964017868041992,
+      "loss": 0.4713,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4989603757858276,
+      "rewards/margins": 0.49744129180908203,
+      "rewards/rejected": -1.9964017868041992,
+      "sft_loss": 1.6010043621063232,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 6.03372519429463,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": -0.15877914428710938,
+      "logits/rejected": -0.040464289486408234,
+      "logps/chosen": -1.6278139352798462,
+      "logps/rejected": -1.9819543361663818,
+      "loss": 0.5118,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6278139352798462,
+      "rewards/margins": 0.3541404604911804,
+      "rewards/rejected": -1.9819543361663818,
+      "sft_loss": 1.699200987815857,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 6.5355149361133895,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": -0.20665928721427917,
+      "logits/rejected": -0.06811292469501495,
+      "logps/chosen": -1.6473109722137451,
+      "logps/rejected": -1.948591947555542,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6473109722137451,
+      "rewards/margins": 0.3012810945510864,
+      "rewards/rejected": -1.948591947555542,
+      "sft_loss": 1.7403411865234375,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 11.784323927993501,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": -0.24116232991218567,
+      "logits/rejected": -0.12919065356254578,
+      "logps/chosen": -1.5015369653701782,
+      "logps/rejected": -1.898349404335022,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5015369653701782,
+      "rewards/margins": 0.3968126177787781,
+      "rewards/rejected": -1.898349404335022,
+      "sft_loss": 1.6438575983047485,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 7.467011155078924,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": -0.18975183367729187,
+      "logits/rejected": -0.050173837691545486,
+      "logps/chosen": -1.5932409763336182,
+      "logps/rejected": -2.0303874015808105,
+      "loss": 0.5,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5932409763336182,
+      "rewards/margins": 0.4371464252471924,
+      "rewards/rejected": -2.0303874015808105,
+      "sft_loss": 1.6250934600830078,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 10.23729026474453,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": -0.24026088416576385,
+      "logits/rejected": -0.1027134507894516,
+      "logps/chosen": -1.456146478652954,
+      "logps/rejected": -1.9496123790740967,
+      "loss": 0.4637,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.456146478652954,
+      "rewards/margins": 0.4934659004211426,
+      "rewards/rejected": -1.9496123790740967,
+      "sft_loss": 1.5960451364517212,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 7.21246606914941,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": -0.16082945466041565,
+      "logits/rejected": -0.03842328116297722,
+      "logps/chosen": -1.398827075958252,
+      "logps/rejected": -1.8935636281967163,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.398827075958252,
+      "rewards/margins": 0.49473652243614197,
+      "rewards/rejected": -1.8935636281967163,
+      "sft_loss": 1.4228825569152832,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 6.769604855335532,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": -0.22607287764549255,
+      "logits/rejected": -0.06951910257339478,
+      "logps/chosen": -1.4958351850509644,
+      "logps/rejected": -2.083021640777588,
+      "loss": 0.4514,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4958351850509644,
+      "rewards/margins": 0.5871865153312683,
+      "rewards/rejected": -2.083021640777588,
+      "sft_loss": 1.6325403451919556,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 7.027012410446907,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": -0.2434675395488739,
+      "logits/rejected": -0.08809126913547516,
+      "logps/chosen": -1.6265697479248047,
+      "logps/rejected": -2.3062081336975098,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6265697479248047,
+      "rewards/margins": 0.6796378493309021,
+      "rewards/rejected": -2.3062081336975098,
+      "sft_loss": 1.6968345642089844,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 10.161618680788564,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": -0.2174391746520996,
+      "logits/rejected": -0.08748964965343475,
+      "logps/chosen": -1.3686556816101074,
+      "logps/rejected": -1.8687818050384521,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3686556816101074,
+      "rewards/margins": 0.5001261234283447,
+      "rewards/rejected": -1.8687818050384521,
+      "sft_loss": 1.4801901578903198,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 8.215662645393254,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": -0.2564537525177002,
+      "logits/rejected": -0.16647759079933167,
+      "logps/chosen": -1.4997940063476562,
+      "logps/rejected": -2.045769214630127,
+      "loss": 0.4809,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4997940063476562,
+      "rewards/margins": 0.5459750890731812,
+      "rewards/rejected": -2.045769214630127,
+      "sft_loss": 1.6288654804229736,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 6.30101921158305,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": -0.20720060169696808,
+      "logits/rejected": -0.036762163043022156,
+      "logps/chosen": -1.5758286714553833,
+      "logps/rejected": -2.0885355472564697,
+      "loss": 0.487,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5758286714553833,
+      "rewards/margins": 0.5127067565917969,
+      "rewards/rejected": -2.0885355472564697,
+      "sft_loss": 1.5761165618896484,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 7.226297337613594,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": -0.15174725651741028,
+      "logits/rejected": -0.09643477201461792,
+      "logps/chosen": -1.5932036638259888,
+      "logps/rejected": -2.095515727996826,
+      "loss": 0.5091,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5932036638259888,
+      "rewards/margins": 0.5023123025894165,
+      "rewards/rejected": -2.095515727996826,
+      "sft_loss": 1.7102028131484985,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 6.184587597685185,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": -0.12151547521352768,
+      "logits/rejected": 0.0030573017429560423,
+      "logps/chosen": -1.4618427753448486,
+      "logps/rejected": -1.9577305316925049,
+      "loss": 0.4717,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4618427753448486,
+      "rewards/margins": 0.4958876669406891,
+      "rewards/rejected": -1.9577305316925049,
+      "sft_loss": 1.5311295986175537,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 6.31477109275785,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": -0.18775346875190735,
+      "logits/rejected": -0.10839760303497314,
+      "logps/chosen": -1.5057272911071777,
+      "logps/rejected": -1.9904956817626953,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5057272911071777,
+      "rewards/margins": 0.4847683906555176,
+      "rewards/rejected": -1.9904956817626953,
+      "sft_loss": 1.5433039665222168,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 8.486572731436723,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": -0.04733508080244064,
+      "logits/rejected": 0.025062087923288345,
+      "logps/chosen": -1.5226056575775146,
+      "logps/rejected": -1.9573904275894165,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5226056575775146,
+      "rewards/margins": 0.43478474020957947,
+      "rewards/rejected": -1.9573904275894165,
+      "sft_loss": 1.640038251876831,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 5.509472701150456,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": -0.14096835255622864,
+      "logits/rejected": -0.06170610338449478,
+      "logps/chosen": -1.582094430923462,
+      "logps/rejected": -2.0880839824676514,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.582094430923462,
+      "rewards/margins": 0.5059898495674133,
+      "rewards/rejected": -2.0880839824676514,
+      "sft_loss": 1.6778017282485962,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 6.147956944803346,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": -0.18321634829044342,
+      "logits/rejected": -0.09819085896015167,
+      "logps/chosen": -1.4959570169448853,
+      "logps/rejected": -1.8988326787948608,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4959570169448853,
+      "rewards/margins": 0.40287572145462036,
+      "rewards/rejected": -1.8988326787948608,
+      "sft_loss": 1.531747817993164,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 5.382806009815984,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": -0.26531511545181274,
+      "logits/rejected": -0.15976980328559875,
+      "logps/chosen": -1.523664951324463,
+      "logps/rejected": -1.9966137409210205,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.523664951324463,
+      "rewards/margins": 0.47294870018959045,
+      "rewards/rejected": -1.9966137409210205,
+      "sft_loss": 1.676461935043335,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 8.441690388912436,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": -0.15571698546409607,
+      "logits/rejected": -0.014239877462387085,
+      "logps/chosen": -1.4310564994812012,
+      "logps/rejected": -1.9260032176971436,
+      "loss": 0.4485,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4310564994812012,
+      "rewards/margins": 0.4949464797973633,
+      "rewards/rejected": -1.9260032176971436,
+      "sft_loss": 1.4834562540054321,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 9.974130353603462,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": -0.1044548749923706,
+      "logits/rejected": -0.014732305891811848,
+      "logps/chosen": -1.598559856414795,
+      "logps/rejected": -1.9291934967041016,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.598559856414795,
+      "rewards/margins": 0.3306335508823395,
+      "rewards/rejected": -1.9291934967041016,
+      "sft_loss": 1.6485782861709595,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 7.78305620035311,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": -0.12677548825740814,
+      "logits/rejected": -0.05798447132110596,
+      "logps/chosen": -1.6210359334945679,
+      "logps/rejected": -1.9703184366226196,
+      "loss": 0.5402,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6210359334945679,
+      "rewards/margins": 0.34928256273269653,
+      "rewards/rejected": -1.9703184366226196,
+      "sft_loss": 1.632920265197754,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 8.74331278022788,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": -0.1670970916748047,
+      "logits/rejected": -0.06462232768535614,
+      "logps/chosen": -1.592901349067688,
+      "logps/rejected": -2.0148890018463135,
+      "loss": 0.5052,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.592901349067688,
+      "rewards/margins": 0.42198771238327026,
+      "rewards/rejected": -2.0148890018463135,
+      "sft_loss": 1.6603543758392334,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 7.576250940089037,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": -0.2261175662279129,
+      "logits/rejected": -0.12932650744915009,
+      "logps/chosen": -1.403714895248413,
+      "logps/rejected": -2.009172201156616,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.403714895248413,
+      "rewards/margins": 0.6054573655128479,
+      "rewards/rejected": -2.009172201156616,
+      "sft_loss": 1.5091825723648071,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 7.69555538626666,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": -0.19793611764907837,
+      "logits/rejected": -0.09182445704936981,
+      "logps/chosen": -1.4750864505767822,
+      "logps/rejected": -2.129772901535034,
+      "loss": 0.4299,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4750864505767822,
+      "rewards/margins": 0.654686450958252,
+      "rewards/rejected": -2.129772901535034,
+      "sft_loss": 1.6487104892730713,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 5.695043205810022,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": -0.2694675624370575,
+      "logits/rejected": -0.22962899506092072,
+      "logps/chosen": -1.5076862573623657,
+      "logps/rejected": -2.1112308502197266,
+      "loss": 0.4655,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5076862573623657,
+      "rewards/margins": 0.6035445928573608,
+      "rewards/rejected": -2.1112308502197266,
+      "sft_loss": 1.6350364685058594,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 11.1683649183914,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": -0.15088486671447754,
+      "logits/rejected": 0.04365579038858414,
+      "logps/chosen": -1.5514955520629883,
+      "logps/rejected": -2.126537799835205,
+      "loss": 0.4801,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5514955520629883,
+      "rewards/margins": 0.5750420689582825,
+      "rewards/rejected": -2.126537799835205,
+      "sft_loss": 1.5818036794662476,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 14.540247275731168,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": -0.2786027491092682,
+      "logits/rejected": -0.10113555192947388,
+      "logps/chosen": -1.4965102672576904,
+      "logps/rejected": -2.2697319984436035,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4965102672576904,
+      "rewards/margins": 0.7732216119766235,
+      "rewards/rejected": -2.2697319984436035,
+      "sft_loss": 1.6020082235336304,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 7.548258945982458,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": -0.22244305908679962,
+      "logits/rejected": -0.15660223364830017,
+      "logps/chosen": -1.5342885255813599,
+      "logps/rejected": -2.0522589683532715,
+      "loss": 0.4753,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5342885255813599,
+      "rewards/margins": 0.517970621585846,
+      "rewards/rejected": -2.0522589683532715,
+      "sft_loss": 1.6077263355255127,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 6.56138098807187,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": -0.18377938866615295,
+      "logits/rejected": -0.13642618060112,
+      "logps/chosen": -1.5458452701568604,
+      "logps/rejected": -2.121659994125366,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5458452701568604,
+      "rewards/margins": 0.575814962387085,
+      "rewards/rejected": -2.121659994125366,
+      "sft_loss": 1.6719255447387695,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 4.89627946113857,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": -0.21539512276649475,
+      "logits/rejected": -0.07043374329805374,
+      "logps/chosen": -1.6168758869171143,
+      "logps/rejected": -2.2531332969665527,
+      "loss": 0.4558,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6168758869171143,
+      "rewards/margins": 0.636257529258728,
+      "rewards/rejected": -2.2531332969665527,
+      "sft_loss": 1.6750949621200562,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 4.515232600959035,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": -0.26013562083244324,
+      "logits/rejected": -0.07724117487668991,
+      "logps/chosen": -1.5244057178497314,
+      "logps/rejected": -2.172802686691284,
+      "loss": 0.4265,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5244057178497314,
+      "rewards/margins": 0.6483969688415527,
+      "rewards/rejected": -2.172802686691284,
+      "sft_loss": 1.6660228967666626,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 7.811442251227995,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": -0.16741736233234406,
+      "logits/rejected": -0.13274990022182465,
+      "logps/chosen": -1.447377324104309,
+      "logps/rejected": -1.9069875478744507,
+      "loss": 0.4801,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.447377324104309,
+      "rewards/margins": 0.45961007475852966,
+      "rewards/rejected": -1.9069875478744507,
+      "sft_loss": 1.583846092224121,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 8.087433317762617,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": -0.173568457365036,
+      "logits/rejected": -0.03578788787126541,
+      "logps/chosen": -1.6015841960906982,
+      "logps/rejected": -2.189645767211914,
+      "loss": 0.4415,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6015841960906982,
+      "rewards/margins": 0.5880613327026367,
+      "rewards/rejected": -2.189645767211914,
+      "sft_loss": 1.7482211589813232,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 6.9505978866225115,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": -0.1564304232597351,
+      "logits/rejected": -0.0258103609085083,
+      "logps/chosen": -1.4258654117584229,
+      "logps/rejected": -1.9665100574493408,
+      "loss": 0.4515,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4258654117584229,
+      "rewards/margins": 0.540644645690918,
+      "rewards/rejected": -1.9665100574493408,
+      "sft_loss": 1.4974491596221924,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 7.860004423872975,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": -0.0722806304693222,
+      "logits/rejected": 0.023296961560845375,
+      "logps/chosen": -1.4594520330429077,
+      "logps/rejected": -2.125936269760132,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4594520330429077,
+      "rewards/margins": 0.6664842963218689,
+      "rewards/rejected": -2.125936269760132,
+      "sft_loss": 1.5164482593536377,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 8.150670426531857,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": -0.2723679840564728,
+      "logits/rejected": -0.1410585194826126,
+      "logps/chosen": -1.539659023284912,
+      "logps/rejected": -2.0731496810913086,
+      "loss": 0.4643,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.539659023284912,
+      "rewards/margins": 0.5334906578063965,
+      "rewards/rejected": -2.0731496810913086,
+      "sft_loss": 1.551286220550537,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 8.78976681786829,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": -0.2862371802330017,
+      "logits/rejected": -0.061904869973659515,
+      "logps/chosen": -1.6095263957977295,
+      "logps/rejected": -2.1704697608947754,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6095263957977295,
+      "rewards/margins": 0.5609431266784668,
+      "rewards/rejected": -2.1704697608947754,
+      "sft_loss": 1.6857903003692627,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 8.155201143271022,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": -0.21058204770088196,
+      "logits/rejected": -0.09403979778289795,
+      "logps/chosen": -1.4634374380111694,
+      "logps/rejected": -2.047750949859619,
+      "loss": 0.4233,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4634374380111694,
+      "rewards/margins": 0.5843135118484497,
+      "rewards/rejected": -2.047750949859619,
+      "sft_loss": 1.5597999095916748,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 6.740274381093628,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": -0.28222018480300903,
+      "logits/rejected": -0.12227698415517807,
+      "logps/chosen": -1.5999635457992554,
+      "logps/rejected": -2.318425178527832,
+      "loss": 0.4205,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5999635457992554,
+      "rewards/margins": 0.7184616923332214,
+      "rewards/rejected": -2.318425178527832,
+      "sft_loss": 1.649235486984253,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 7.080030593128097,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": -0.19885589182376862,
+      "logits/rejected": -0.020628320053219795,
+      "logps/chosen": -1.5972665548324585,
+      "logps/rejected": -2.2121667861938477,
+      "loss": 0.4569,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5972665548324585,
+      "rewards/margins": 0.6149001717567444,
+      "rewards/rejected": -2.2121667861938477,
+      "sft_loss": 1.6844546794891357,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 5.652564618824891,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": -0.1308698207139969,
+      "logits/rejected": -0.0033002153504639864,
+      "logps/chosen": -1.4605486392974854,
+      "logps/rejected": -2.12688946723938,
+      "loss": 0.428,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4605486392974854,
+      "rewards/margins": 0.6663408279418945,
+      "rewards/rejected": -2.12688946723938,
+      "sft_loss": 1.5540508031845093,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 4.777212634284139,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": -0.33631742000579834,
+      "logits/rejected": -0.0764494314789772,
+      "logps/chosen": -1.634876012802124,
+      "logps/rejected": -2.2439773082733154,
+      "loss": 0.4502,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.634876012802124,
+      "rewards/margins": 0.609101414680481,
+      "rewards/rejected": -2.2439773082733154,
+      "sft_loss": 1.6806738376617432,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 6.863100762156229,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": -0.2605910897254944,
+      "logits/rejected": -0.08244252949953079,
+      "logps/chosen": -1.6415363550186157,
+      "logps/rejected": -2.2351536750793457,
+      "loss": 0.474,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6415363550186157,
+      "rewards/margins": 0.5936172604560852,
+      "rewards/rejected": -2.2351536750793457,
+      "sft_loss": 1.7238140106201172,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 13.936388029108453,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": -0.18211022019386292,
+      "logits/rejected": 0.006972149014472961,
+      "logps/chosen": -1.6002590656280518,
+      "logps/rejected": -2.4544899463653564,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6002590656280518,
+      "rewards/margins": 0.8542307615280151,
+      "rewards/rejected": -2.4544899463653564,
+      "sft_loss": 1.63395094871521,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 9.576481319148407,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": -0.214491605758667,
+      "logits/rejected": -0.056644368916749954,
+      "logps/chosen": -1.5494184494018555,
+      "logps/rejected": -2.272798538208008,
+      "loss": 0.4357,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5494184494018555,
+      "rewards/margins": 0.7233801484107971,
+      "rewards/rejected": -2.272798538208008,
+      "sft_loss": 1.5816377401351929,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 10.928117257713213,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": -0.31673842668533325,
+      "logits/rejected": -0.11347083747386932,
+      "logps/chosen": -1.6189310550689697,
+      "logps/rejected": -2.098507881164551,
+      "loss": 0.498,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6189310550689697,
+      "rewards/margins": 0.4795767664909363,
+      "rewards/rejected": -2.098507881164551,
+      "sft_loss": 1.7159723043441772,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 6.5000832860610025,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": -0.2351851910352707,
+      "logits/rejected": -0.16655540466308594,
+      "logps/chosen": -1.4487316608428955,
+      "logps/rejected": -2.002185583114624,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4487316608428955,
+      "rewards/margins": 0.5534540414810181,
+      "rewards/rejected": -2.002185583114624,
+      "sft_loss": 1.5564430952072144,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 7.302439144857689,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": -0.2277861088514328,
+      "logits/rejected": -0.11319579929113388,
+      "logps/chosen": -1.6114695072174072,
+      "logps/rejected": -2.214570999145508,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6114695072174072,
+      "rewards/margins": 0.6031014323234558,
+      "rewards/rejected": -2.214570999145508,
+      "sft_loss": 1.7454404830932617,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 11.473067531134857,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": -0.21884004771709442,
+      "logits/rejected": -0.036847881972789764,
+      "logps/chosen": -1.3591184616088867,
+      "logps/rejected": -2.0386531352996826,
+      "loss": 0.402,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3591184616088867,
+      "rewards/margins": 0.6795347332954407,
+      "rewards/rejected": -2.0386531352996826,
+      "sft_loss": 1.4358450174331665,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 7.667532299861024,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": -0.26414209604263306,
+      "logits/rejected": -0.100743368268013,
+      "logps/chosen": -1.4504120349884033,
+      "logps/rejected": -2.148660898208618,
+      "loss": 0.4313,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4504120349884033,
+      "rewards/margins": 0.698248565196991,
+      "rewards/rejected": -2.148660898208618,
+      "sft_loss": 1.566084623336792,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 6.60739297994297,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": -0.12682762742042542,
+      "logits/rejected": -0.027794640511274338,
+      "logps/chosen": -1.5632338523864746,
+      "logps/rejected": -2.1642110347747803,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5632338523864746,
+      "rewards/margins": 0.6009770631790161,
+      "rewards/rejected": -2.1642110347747803,
+      "sft_loss": 1.6938755512237549,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 6.770016103035415,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": -0.2298419028520584,
+      "logits/rejected": 0.02210015431046486,
+      "logps/chosen": -1.5325865745544434,
+      "logps/rejected": -2.1005892753601074,
+      "loss": 0.475,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5325865745544434,
+      "rewards/margins": 0.568002462387085,
+      "rewards/rejected": -2.1005892753601074,
+      "sft_loss": 1.596083641052246,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 12.56192636846323,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": -0.10446188598871231,
+      "logits/rejected": -0.08038032799959183,
+      "logps/chosen": -1.5202323198318481,
+      "logps/rejected": -2.0459377765655518,
+      "loss": 0.4598,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5202323198318481,
+      "rewards/margins": 0.5257053375244141,
+      "rewards/rejected": -2.0459377765655518,
+      "sft_loss": 1.617698311805725,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 6.756218665882484,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": -0.17761339247226715,
+      "logits/rejected": -0.07439003139734268,
+      "logps/chosen": -1.5598061084747314,
+      "logps/rejected": -2.0722603797912598,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5598061084747314,
+      "rewards/margins": 0.5124542117118835,
+      "rewards/rejected": -2.0722603797912598,
+      "sft_loss": 1.5847612619400024,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 6.298927224311675,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": -0.13386814296245575,
+      "logits/rejected": 0.00870521366596222,
+      "logps/chosen": -1.3839236497879028,
+      "logps/rejected": -1.9484390020370483,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3839236497879028,
+      "rewards/margins": 0.5645155310630798,
+      "rewards/rejected": -1.9484390020370483,
+      "sft_loss": 1.4806008338928223,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 8.760310055144243,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": -0.2275843620300293,
+      "logits/rejected": -0.043379418551921844,
+      "logps/chosen": -1.5875427722930908,
+      "logps/rejected": -2.2296032905578613,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5875427722930908,
+      "rewards/margins": 0.6420606970787048,
+      "rewards/rejected": -2.2296032905578613,
+      "sft_loss": 1.6525790691375732,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 9.378349348436625,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": -0.21023735404014587,
+      "logits/rejected": -0.03672776371240616,
+      "logps/chosen": -1.5500128269195557,
+      "logps/rejected": -2.308952808380127,
+      "loss": 0.4324,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.5500128269195557,
+      "rewards/margins": 0.7589399814605713,
+      "rewards/rejected": -2.308952808380127,
+      "sft_loss": 1.6003620624542236,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 6.685879334267199,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": -0.20083554089069366,
+      "logits/rejected": -0.08228044211864471,
+      "logps/chosen": -1.4767929315567017,
+      "logps/rejected": -2.145826816558838,
+      "loss": 0.4338,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4767929315567017,
+      "rewards/margins": 0.6690336465835571,
+      "rewards/rejected": -2.145826816558838,
+      "sft_loss": 1.572529911994934,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 5.737150130242227,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": -0.290099561214447,
+      "logits/rejected": -0.12537804245948792,
+      "logps/chosen": -1.524739384651184,
+      "logps/rejected": -2.1593596935272217,
+      "loss": 0.4434,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.524739384651184,
+      "rewards/margins": 0.6346203684806824,
+      "rewards/rejected": -2.1593596935272217,
+      "sft_loss": 1.5483014583587646,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 6.2794896545930765,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": -0.17081448435783386,
+      "logits/rejected": -0.10958276689052582,
+      "logps/chosen": -1.560088872909546,
+      "logps/rejected": -2.201547622680664,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.560088872909546,
+      "rewards/margins": 0.6414587497711182,
+      "rewards/rejected": -2.201547622680664,
+      "sft_loss": 1.574739694595337,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 10.42380917965788,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": -0.2000219076871872,
+      "logits/rejected": -0.1316196471452713,
+      "logps/chosen": -1.4703319072723389,
+      "logps/rejected": -2.215935707092285,
+      "loss": 0.4202,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4703319072723389,
+      "rewards/margins": 0.7456039786338806,
+      "rewards/rejected": -2.215935707092285,
+      "sft_loss": 1.495848298072815,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 10.759877860016946,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": -0.21619954705238342,
+      "logits/rejected": -0.03465006873011589,
+      "logps/chosen": -1.6587110757827759,
+      "logps/rejected": -2.1734070777893066,
+      "loss": 0.5023,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6587110757827759,
+      "rewards/margins": 0.5146957635879517,
+      "rewards/rejected": -2.1734070777893066,
+      "sft_loss": 1.7548824548721313,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 11.314688619454033,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": -0.12167392671108246,
+      "logits/rejected": 0.04851360246539116,
+      "logps/chosen": -1.4907797574996948,
+      "logps/rejected": -2.0790886878967285,
+      "loss": 0.4597,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4907797574996948,
+      "rewards/margins": 0.5883088707923889,
+      "rewards/rejected": -2.0790886878967285,
+      "sft_loss": 1.560671091079712,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 6.600920678232209,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": -0.2696961462497711,
+      "logits/rejected": -0.17066015303134918,
+      "logps/chosen": -1.4763388633728027,
+      "logps/rejected": -2.1660704612731934,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4763388633728027,
+      "rewards/margins": 0.6897314786911011,
+      "rewards/rejected": -2.1660704612731934,
+      "sft_loss": 1.5858443975448608,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 10.608197576928287,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": -0.14795371890068054,
+      "logits/rejected": -0.02199488878250122,
+      "logps/chosen": -1.5266574621200562,
+      "logps/rejected": -2.2974257469177246,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5266574621200562,
+      "rewards/margins": 0.7707680463790894,
+      "rewards/rejected": -2.2974257469177246,
+      "sft_loss": 1.6296428442001343,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 7.218085338433899,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": -0.14386795461177826,
+      "logits/rejected": -0.024950990453362465,
+      "logps/chosen": -1.5040401220321655,
+      "logps/rejected": -2.217085361480713,
+      "loss": 0.4303,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.5040401220321655,
+      "rewards/margins": 0.7130452990531921,
+      "rewards/rejected": -2.217085361480713,
+      "sft_loss": 1.6567966938018799,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 6.661608023455321,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": -0.17446637153625488,
+      "logits/rejected": -0.06813861429691315,
+      "logps/chosen": -1.454469919204712,
+      "logps/rejected": -2.1430160999298096,
+      "loss": 0.4389,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.454469919204712,
+      "rewards/margins": 0.6885461807250977,
+      "rewards/rejected": -2.1430160999298096,
+      "sft_loss": 1.603070616722107,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 9.347508736323672,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": -0.1273220181465149,
+      "logits/rejected": -0.019476449117064476,
+      "logps/chosen": -1.3374217748641968,
+      "logps/rejected": -1.8788425922393799,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3374217748641968,
+      "rewards/margins": 0.5414208173751831,
+      "rewards/rejected": -1.8788425922393799,
+      "sft_loss": 1.3960545063018799,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 6.201290406117352,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": -0.2582013010978699,
+      "logits/rejected": -0.1725609004497528,
+      "logps/chosen": -1.3875207901000977,
+      "logps/rejected": -2.1059584617614746,
+      "loss": 0.4121,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3875207901000977,
+      "rewards/margins": 0.7184377908706665,
+      "rewards/rejected": -2.1059584617614746,
+      "sft_loss": 1.572930932044983,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 9.37812664861454,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": -0.26035767793655396,
+      "logits/rejected": -0.14250019192695618,
+      "logps/chosen": -1.5116602182388306,
+      "logps/rejected": -2.161254644393921,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5116602182388306,
+      "rewards/margins": 0.6495946645736694,
+      "rewards/rejected": -2.161254644393921,
+      "sft_loss": 1.6295945644378662,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 7.841514997852571,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": -0.31633514165878296,
+      "logits/rejected": -0.17246940732002258,
+      "logps/chosen": -1.4053174257278442,
+      "logps/rejected": -2.0792293548583984,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4053174257278442,
+      "rewards/margins": 0.6739118695259094,
+      "rewards/rejected": -2.0792293548583984,
+      "sft_loss": 1.5280014276504517,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 13.6967942623219,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": -0.35641390085220337,
+      "logits/rejected": -0.08034927397966385,
+      "logps/chosen": -1.5294626951217651,
+      "logps/rejected": -2.208108425140381,
+      "loss": 0.4388,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5294626951217651,
+      "rewards/margins": 0.6786457896232605,
+      "rewards/rejected": -2.208108425140381,
+      "sft_loss": 1.6018537282943726,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 11.98212137009441,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": -0.21660706400871277,
+      "logits/rejected": -0.10180102288722992,
+      "logps/chosen": -1.4509875774383545,
+      "logps/rejected": -2.1792187690734863,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4509875774383545,
+      "rewards/margins": 0.7282313108444214,
+      "rewards/rejected": -2.1792187690734863,
+      "sft_loss": 1.5786430835723877,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": 0.13109426200389862,
+      "eval_logits/rejected": 0.23107898235321045,
+      "eval_logps/chosen": -1.5826647281646729,
+      "eval_logps/rejected": -2.126399278640747,
+      "eval_loss": 0.4894164204597473,
+      "eval_rewards/accuracies": 0.669881284236908,
+      "eval_rewards/chosen": -1.5826647281646729,
+      "eval_rewards/margins": 0.5437345504760742,
+      "eval_rewards/rejected": -2.126399278640747,
+      "eval_runtime": 43.266,
+      "eval_samples_per_second": 31.087,
+      "eval_sft_loss": 1.6623671054840088,
+      "eval_steps_per_second": 7.789,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 6.774209271920777,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": -0.25786370038986206,
+      "logits/rejected": -0.03186394274234772,
+      "logps/chosen": -1.7039791345596313,
+      "logps/rejected": -2.3183369636535645,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7039791345596313,
+      "rewards/margins": 0.6143580079078674,
+      "rewards/rejected": -2.3183369636535645,
+      "sft_loss": 1.7434800863265991,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 11.204427632919344,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": -0.25920170545578003,
+      "logits/rejected": -0.14853177964687347,
+      "logps/chosen": -1.5233911275863647,
+      "logps/rejected": -2.025580883026123,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5233911275863647,
+      "rewards/margins": 0.5021899938583374,
+      "rewards/rejected": -2.025580883026123,
+      "sft_loss": 1.608466386795044,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 6.277393465452174,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": -0.250881552696228,
+      "logits/rejected": 0.021695155650377274,
+      "logps/chosen": -1.5364277362823486,
+      "logps/rejected": -2.1099002361297607,
+      "loss": 0.4782,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5364277362823486,
+      "rewards/margins": 0.5734724402427673,
+      "rewards/rejected": -2.1099002361297607,
+      "sft_loss": 1.6285278797149658,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 10.927604344319644,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": -0.35845470428466797,
+      "logits/rejected": -0.19380858540534973,
+      "logps/chosen": -1.6200428009033203,
+      "logps/rejected": -2.2093708515167236,
+      "loss": 0.4705,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6200428009033203,
+      "rewards/margins": 0.5893279314041138,
+      "rewards/rejected": -2.2093708515167236,
+      "sft_loss": 1.7426955699920654,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 7.323032772336199,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": -0.22688297927379608,
+      "logits/rejected": -0.11204588413238525,
+      "logps/chosen": -1.4713099002838135,
+      "logps/rejected": -2.0694479942321777,
+      "loss": 0.4489,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4713099002838135,
+      "rewards/margins": 0.5981377363204956,
+      "rewards/rejected": -2.0694479942321777,
+      "sft_loss": 1.527937650680542,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 14.434754823938734,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": -0.2611876130104065,
+      "logits/rejected": -0.16281744837760925,
+      "logps/chosen": -1.4556105136871338,
+      "logps/rejected": -2.102907657623291,
+      "loss": 0.4324,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4556105136871338,
+      "rewards/margins": 0.6472971439361572,
+      "rewards/rejected": -2.102907657623291,
+      "sft_loss": 1.5691444873809814,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 9.831723652563838,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": -0.24293653666973114,
+      "logits/rejected": -0.19557669758796692,
+      "logps/chosen": -1.4355570077896118,
+      "logps/rejected": -1.8510463237762451,
+      "loss": 0.4623,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4355570077896118,
+      "rewards/margins": 0.4154892861843109,
+      "rewards/rejected": -1.8510463237762451,
+      "sft_loss": 1.569411277770996,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 7.351491599536689,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": -0.25733011960983276,
+      "logits/rejected": -0.21016161143779755,
+      "logps/chosen": -1.3895686864852905,
+      "logps/rejected": -1.9499037265777588,
+      "loss": 0.4579,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3895686864852905,
+      "rewards/margins": 0.5603350400924683,
+      "rewards/rejected": -1.9499037265777588,
+      "sft_loss": 1.4448275566101074,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 8.226102757499863,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": -0.18892326951026917,
+      "logits/rejected": -0.049190543591976166,
+      "logps/chosen": -1.52090585231781,
+      "logps/rejected": -2.069546699523926,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.52090585231781,
+      "rewards/margins": 0.5486409664154053,
+      "rewards/rejected": -2.069546699523926,
+      "sft_loss": 1.6149247884750366,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 7.705027333491286,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": -0.2961641252040863,
+      "logits/rejected": -0.18423901498317719,
+      "logps/chosen": -1.5554035902023315,
+      "logps/rejected": -2.0131006240844727,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5554035902023315,
+      "rewards/margins": 0.4576972424983978,
+      "rewards/rejected": -2.0131006240844727,
+      "sft_loss": 1.6668485403060913,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 8.141312762737401,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": -0.18819668889045715,
+      "logits/rejected": -0.13292866945266724,
+      "logps/chosen": -1.5318689346313477,
+      "logps/rejected": -1.9563630819320679,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5318689346313477,
+      "rewards/margins": 0.42449426651000977,
+      "rewards/rejected": -1.9563630819320679,
+      "sft_loss": 1.5768083333969116,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 10.039453100753366,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": -0.09409301728010178,
+      "logits/rejected": 0.020703068003058434,
+      "logps/chosen": -1.436989426612854,
+      "logps/rejected": -1.986651062965393,
+      "loss": 0.4476,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.436989426612854,
+      "rewards/margins": 0.5496617555618286,
+      "rewards/rejected": -1.986651062965393,
+      "sft_loss": 1.5771474838256836,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 8.027729882763833,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": -0.31718340516090393,
+      "logits/rejected": -0.21360819041728973,
+      "logps/chosen": -1.5581392049789429,
+      "logps/rejected": -2.234459400177002,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.5581392049789429,
+      "rewards/margins": 0.6763203144073486,
+      "rewards/rejected": -2.234459400177002,
+      "sft_loss": 1.6486304998397827,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 10.71593597455729,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": -0.20124073326587677,
+      "logits/rejected": 0.020577391609549522,
+      "logps/chosen": -1.5547935962677002,
+      "logps/rejected": -2.3254096508026123,
+      "loss": 0.4472,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5547935962677002,
+      "rewards/margins": 0.7706161141395569,
+      "rewards/rejected": -2.3254096508026123,
+      "sft_loss": 1.7266238927841187,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 9.236529459942533,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": -0.27028411626815796,
+      "logits/rejected": -0.06531749665737152,
+      "logps/chosen": -1.6120535135269165,
+      "logps/rejected": -2.1328024864196777,
+      "loss": 0.4762,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6120535135269165,
+      "rewards/margins": 0.5207492113113403,
+      "rewards/rejected": -2.1328024864196777,
+      "sft_loss": 1.6699886322021484,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 7.3321692955545545,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": -0.243181511759758,
+      "logits/rejected": -0.09978292882442474,
+      "logps/chosen": -1.5708192586898804,
+      "logps/rejected": -2.295379638671875,
+      "loss": 0.4431,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5708192586898804,
+      "rewards/margins": 0.7245603203773499,
+      "rewards/rejected": -2.295379638671875,
+      "sft_loss": 1.692696213722229,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 9.164383434433807,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": -0.22102339565753937,
+      "logits/rejected": -0.11209341138601303,
+      "logps/chosen": -1.4797003269195557,
+      "logps/rejected": -2.004822254180908,
+      "loss": 0.4526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4797003269195557,
+      "rewards/margins": 0.5251215696334839,
+      "rewards/rejected": -2.004822254180908,
+      "sft_loss": 1.5114164352416992,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 9.347179709940297,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": -0.30683061480522156,
+      "logits/rejected": -0.09695632755756378,
+      "logps/chosen": -1.4971176385879517,
+      "logps/rejected": -2.1254069805145264,
+      "loss": 0.4462,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4971176385879517,
+      "rewards/margins": 0.6282894611358643,
+      "rewards/rejected": -2.1254069805145264,
+      "sft_loss": 1.6113370656967163,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 5.754956838679529,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": -0.2652248740196228,
+      "logits/rejected": -0.1713368147611618,
+      "logps/chosen": -1.588734745979309,
+      "logps/rejected": -2.086512804031372,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.588734745979309,
+      "rewards/margins": 0.4977780878543854,
+      "rewards/rejected": -2.086512804031372,
+      "sft_loss": 1.6886154413223267,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 7.582408807269065,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": -0.19961073994636536,
+      "logits/rejected": -0.05144035071134567,
+      "logps/chosen": -1.4022175073623657,
+      "logps/rejected": -2.1321256160736084,
+      "loss": 0.407,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4022175073623657,
+      "rewards/margins": 0.7299081683158875,
+      "rewards/rejected": -2.1321256160736084,
+      "sft_loss": 1.5723204612731934,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 7.244622856694702,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": -0.23503056168556213,
+      "logits/rejected": -0.02701449953019619,
+      "logps/chosen": -1.6697590351104736,
+      "logps/rejected": -2.253340482711792,
+      "loss": 0.482,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6697590351104736,
+      "rewards/margins": 0.583581268787384,
+      "rewards/rejected": -2.253340482711792,
+      "sft_loss": 1.7181288003921509,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 16.411061274639884,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": -0.20764736831188202,
+      "logits/rejected": -0.016535207629203796,
+      "logps/chosen": -1.5405995845794678,
+      "logps/rejected": -2.188555955886841,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5405995845794678,
+      "rewards/margins": 0.647956371307373,
+      "rewards/rejected": -2.188555955886841,
+      "sft_loss": 1.6755081415176392,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 10.865795852770049,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": -0.18737569451332092,
+      "logits/rejected": -0.12781764566898346,
+      "logps/chosen": -1.585011601448059,
+      "logps/rejected": -2.272808074951172,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.585011601448059,
+      "rewards/margins": 0.6877964735031128,
+      "rewards/rejected": -2.272808074951172,
+      "sft_loss": 1.6630890369415283,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 8.067074691179583,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": -0.22606685757637024,
+      "logits/rejected": -0.1338723599910736,
+      "logps/chosen": -1.4738590717315674,
+      "logps/rejected": -2.151301622390747,
+      "loss": 0.4325,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4738590717315674,
+      "rewards/margins": 0.6774424314498901,
+      "rewards/rejected": -2.151301622390747,
+      "sft_loss": 1.6365344524383545,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 6.89501685383988,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": -0.2842291295528412,
+      "logits/rejected": -0.04498632252216339,
+      "logps/chosen": -1.5477343797683716,
+      "logps/rejected": -2.343984842300415,
+      "loss": 0.4111,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5477343797683716,
+      "rewards/margins": 0.7962502241134644,
+      "rewards/rejected": -2.343984842300415,
+      "sft_loss": 1.683488130569458,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 12.959262244969201,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": -0.19628119468688965,
+      "logits/rejected": -0.10231286287307739,
+      "logps/chosen": -1.5733191967010498,
+      "logps/rejected": -2.239591360092163,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5733191967010498,
+      "rewards/margins": 0.6662724614143372,
+      "rewards/rejected": -2.239591360092163,
+      "sft_loss": 1.6991240978240967,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 8.011736146801448,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": -0.189265638589859,
+      "logits/rejected": -0.18786805868148804,
+      "logps/chosen": -1.6608617305755615,
+      "logps/rejected": -2.460606098175049,
+      "loss": 0.4715,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6608617305755615,
+      "rewards/margins": 0.7997443675994873,
+      "rewards/rejected": -2.460606098175049,
+      "sft_loss": 1.7233469486236572,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 10.22155169357895,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": -0.34872856736183167,
+      "logits/rejected": -0.25761353969573975,
+      "logps/chosen": -1.514994740486145,
+      "logps/rejected": -2.1406776905059814,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.514994740486145,
+      "rewards/margins": 0.6256829500198364,
+      "rewards/rejected": -2.1406776905059814,
+      "sft_loss": 1.6128156185150146,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 8.166285788125295,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": -0.13309597969055176,
+      "logits/rejected": 0.01610729657113552,
+      "logps/chosen": -1.6787502765655518,
+      "logps/rejected": -2.3410792350769043,
+      "loss": 0.4649,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6787502765655518,
+      "rewards/margins": 0.6623291373252869,
+      "rewards/rejected": -2.3410792350769043,
+      "sft_loss": 1.7590233087539673,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 6.692670737186019,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": -0.18270069360733032,
+      "logits/rejected": -0.07625246793031693,
+      "logps/chosen": -1.4268088340759277,
+      "logps/rejected": -2.050219774246216,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4268088340759277,
+      "rewards/margins": 0.6234109997749329,
+      "rewards/rejected": -2.050219774246216,
+      "sft_loss": 1.5354491472244263,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 9.108793827152095,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": -0.1617237627506256,
+      "logits/rejected": -0.17442257702350616,
+      "logps/chosen": -1.5069429874420166,
+      "logps/rejected": -2.02146053314209,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5069429874420166,
+      "rewards/margins": 0.5145175457000732,
+      "rewards/rejected": -2.02146053314209,
+      "sft_loss": 1.567132592201233,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 10.438365653518229,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": -0.20505666732788086,
+      "logits/rejected": -0.040448617190122604,
+      "logps/chosen": -1.6469700336456299,
+      "logps/rejected": -2.1686878204345703,
+      "loss": 0.4657,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6469700336456299,
+      "rewards/margins": 0.5217178463935852,
+      "rewards/rejected": -2.1686878204345703,
+      "sft_loss": 1.7205617427825928,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 11.070803534114043,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": -0.2014208287000656,
+      "logits/rejected": -0.06682918220758438,
+      "logps/chosen": -1.5332292318344116,
+      "logps/rejected": -2.188716173171997,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5332292318344116,
+      "rewards/margins": 0.6554868817329407,
+      "rewards/rejected": -2.188716173171997,
+      "sft_loss": 1.6469694375991821,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 9.181331373544161,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": -0.09014538675546646,
+      "logits/rejected": -0.01346281636506319,
+      "logps/chosen": -1.5006886720657349,
+      "logps/rejected": -2.145022392272949,
+      "loss": 0.4344,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5006886720657349,
+      "rewards/margins": 0.6443338394165039,
+      "rewards/rejected": -2.145022392272949,
+      "sft_loss": 1.5990290641784668,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 12.213842457899116,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": -0.219885915517807,
+      "logits/rejected": -0.09703179448843002,
+      "logps/chosen": -1.518066644668579,
+      "logps/rejected": -1.9615545272827148,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.518066644668579,
+      "rewards/margins": 0.4434877336025238,
+      "rewards/rejected": -1.9615545272827148,
+      "sft_loss": 1.5732790231704712,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 10.116284053671981,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": -0.23928925395011902,
+      "logits/rejected": 0.006835061125457287,
+      "logps/chosen": -1.4725453853607178,
+      "logps/rejected": -2.2228915691375732,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4725453853607178,
+      "rewards/margins": 0.7503459453582764,
+      "rewards/rejected": -2.2228915691375732,
+      "sft_loss": 1.584643006324768,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 8.807905961289755,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": -0.27372652292251587,
+      "logits/rejected": -0.2470136135816574,
+      "logps/chosen": -1.4996545314788818,
+      "logps/rejected": -2.1062915325164795,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4996545314788818,
+      "rewards/margins": 0.6066369414329529,
+      "rewards/rejected": -2.1062915325164795,
+      "sft_loss": 1.6074802875518799,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 6.763198406992284,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": -0.11524118483066559,
+      "logits/rejected": 0.0006000146386213601,
+      "logps/chosen": -1.4346461296081543,
+      "logps/rejected": -2.095411539077759,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4346461296081543,
+      "rewards/margins": 0.6607652902603149,
+      "rewards/rejected": -2.095411539077759,
+      "sft_loss": 1.534780502319336,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 5.831391069472206,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": -0.32675114274024963,
+      "logits/rejected": -0.1725514978170395,
+      "logps/chosen": -1.5440665483474731,
+      "logps/rejected": -2.0298123359680176,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5440665483474731,
+      "rewards/margins": 0.48574599623680115,
+      "rewards/rejected": -2.0298123359680176,
+      "sft_loss": 1.6876633167266846,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 8.354466364764948,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": -0.08029486984014511,
+      "logits/rejected": -0.09093089401721954,
+      "logps/chosen": -1.6234248876571655,
+      "logps/rejected": -2.1632299423217773,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6234248876571655,
+      "rewards/margins": 0.5398050546646118,
+      "rewards/rejected": -2.1632299423217773,
+      "sft_loss": 1.6912288665771484,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 6.8242452196900185,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": -0.2887396812438965,
+      "logits/rejected": -0.21421535313129425,
+      "logps/chosen": -1.4559670686721802,
+      "logps/rejected": -2.0635907649993896,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4559670686721802,
+      "rewards/margins": 0.607623815536499,
+      "rewards/rejected": -2.0635907649993896,
+      "sft_loss": 1.5633571147918701,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 5.827910714951243,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": -0.23325955867767334,
+      "logits/rejected": -0.10476813465356827,
+      "logps/chosen": -1.5497891902923584,
+      "logps/rejected": -2.382882595062256,
+      "loss": 0.4267,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5497891902923584,
+      "rewards/margins": 0.8330934643745422,
+      "rewards/rejected": -2.382882595062256,
+      "sft_loss": 1.684547781944275,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 9.32355207297127,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": -0.23333308100700378,
+      "logits/rejected": -0.07826565951108932,
+      "logps/chosen": -1.505821704864502,
+      "logps/rejected": -2.1882753372192383,
+      "loss": 0.4583,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.505821704864502,
+      "rewards/margins": 0.6824537515640259,
+      "rewards/rejected": -2.1882753372192383,
+      "sft_loss": 1.5673660039901733,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 8.015105705559941,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": -0.2862780690193176,
+      "logits/rejected": -0.06725447624921799,
+      "logps/chosen": -1.5083166360855103,
+      "logps/rejected": -2.1370108127593994,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5083166360855103,
+      "rewards/margins": 0.6286943554878235,
+      "rewards/rejected": -2.1370108127593994,
+      "sft_loss": 1.6248115301132202,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 7.8642532134225815,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": -0.277448832988739,
+      "logits/rejected": -0.07319362461566925,
+      "logps/chosen": -1.5987507104873657,
+      "logps/rejected": -2.286151170730591,
+      "loss": 0.4288,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5987507104873657,
+      "rewards/margins": 0.6874004602432251,
+      "rewards/rejected": -2.286151170730591,
+      "sft_loss": 1.6796283721923828,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 6.92372534786143,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": -0.26496168971061707,
+      "logits/rejected": -0.15971460938453674,
+      "logps/chosen": -1.5722956657409668,
+      "logps/rejected": -2.2052111625671387,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5722956657409668,
+      "rewards/margins": 0.6329156160354614,
+      "rewards/rejected": -2.2052111625671387,
+      "sft_loss": 1.6152551174163818,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 6.258699276893173,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": -0.19774913787841797,
+      "logits/rejected": -0.07455114275217056,
+      "logps/chosen": -1.5831193923950195,
+      "logps/rejected": -2.035914897918701,
+      "loss": 0.505,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5831193923950195,
+      "rewards/margins": 0.45279568433761597,
+      "rewards/rejected": -2.035914897918701,
+      "sft_loss": 1.7011892795562744,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 7.605306577163764,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": -0.229135662317276,
+      "logits/rejected": -0.0910092145204544,
+      "logps/chosen": -1.5330448150634766,
+      "logps/rejected": -2.0770211219787598,
+      "loss": 0.4496,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5330448150634766,
+      "rewards/margins": 0.5439764261245728,
+      "rewards/rejected": -2.0770211219787598,
+      "sft_loss": 1.6638119220733643,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 10.699895417760684,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": -0.2965095341205597,
+      "logits/rejected": -0.1600067913532257,
+      "logps/chosen": -1.6080896854400635,
+      "logps/rejected": -2.174994945526123,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6080896854400635,
+      "rewards/margins": 0.5669052600860596,
+      "rewards/rejected": -2.174994945526123,
+      "sft_loss": 1.7067543268203735,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 6.405918147574539,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": -0.21542322635650635,
+      "logits/rejected": -0.010124053806066513,
+      "logps/chosen": -1.629784345626831,
+      "logps/rejected": -2.248307943344116,
+      "loss": 0.4734,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.629784345626831,
+      "rewards/margins": 0.6185236573219299,
+      "rewards/rejected": -2.248307943344116,
+      "sft_loss": 1.686134696006775,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 11.065224789631493,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": -0.24650788307189941,
+      "logits/rejected": 0.009222614578902721,
+      "logps/chosen": -1.6138334274291992,
+      "logps/rejected": -2.3137295246124268,
+      "loss": 0.4455,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6138334274291992,
+      "rewards/margins": 0.6998960375785828,
+      "rewards/rejected": -2.3137295246124268,
+      "sft_loss": 1.6994917392730713,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 7.584697213644501,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": -0.21565786004066467,
+      "logits/rejected": -0.09113094955682755,
+      "logps/chosen": -1.461003303527832,
+      "logps/rejected": -2.0535426139831543,
+      "loss": 0.4331,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.461003303527832,
+      "rewards/margins": 0.592539370059967,
+      "rewards/rejected": -2.0535426139831543,
+      "sft_loss": 1.4886611700057983,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 6.263987411806834,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": -0.3257189989089966,
+      "logits/rejected": -0.13461880385875702,
+      "logps/chosen": -1.522209644317627,
+      "logps/rejected": -2.2624077796936035,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.522209644317627,
+      "rewards/margins": 0.7401978373527527,
+      "rewards/rejected": -2.2624077796936035,
+      "sft_loss": 1.6207393407821655,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 7.356539401543046,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": -0.12847329676151276,
+      "logits/rejected": 0.13997411727905273,
+      "logps/chosen": -1.586727261543274,
+      "logps/rejected": -2.3713271617889404,
+      "loss": 0.4369,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.586727261543274,
+      "rewards/margins": 0.7845998406410217,
+      "rewards/rejected": -2.3713271617889404,
+      "sft_loss": 1.6232751607894897,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 8.907757600140817,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": -0.17414642870426178,
+      "logits/rejected": -0.06463111937046051,
+      "logps/chosen": -1.633993148803711,
+      "logps/rejected": -2.227522611618042,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.633993148803711,
+      "rewards/margins": 0.5935293436050415,
+      "rewards/rejected": -2.227522611618042,
+      "sft_loss": 1.7294479608535767,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 11.731433072211523,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": -0.2536371648311615,
+      "logits/rejected": -0.1423775851726532,
+      "logps/chosen": -1.4991668462753296,
+      "logps/rejected": -2.1331608295440674,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4991668462753296,
+      "rewards/margins": 0.6339941024780273,
+      "rewards/rejected": -2.1331608295440674,
+      "sft_loss": 1.607783317565918,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 12.406651356740806,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": -0.13887982070446014,
+      "logits/rejected": -0.1457427442073822,
+      "logps/chosen": -1.4876251220703125,
+      "logps/rejected": -2.082301139831543,
+      "loss": 0.453,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4876251220703125,
+      "rewards/margins": 0.59467613697052,
+      "rewards/rejected": -2.082301139831543,
+      "sft_loss": 1.6114146709442139,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 8.930338489655787,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": -0.27472439408302307,
+      "logits/rejected": -0.09401975572109222,
+      "logps/chosen": -1.6199219226837158,
+      "logps/rejected": -2.2362000942230225,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6199219226837158,
+      "rewards/margins": 0.616278350353241,
+      "rewards/rejected": -2.2362000942230225,
+      "sft_loss": 1.6935663223266602,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 10.706568122216572,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": -0.24929042160511017,
+      "logits/rejected": -0.18465907871723175,
+      "logps/chosen": -1.5443518161773682,
+      "logps/rejected": -2.1019937992095947,
+      "loss": 0.4599,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5443518161773682,
+      "rewards/margins": 0.5576421022415161,
+      "rewards/rejected": -2.1019937992095947,
+      "sft_loss": 1.6194578409194946,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 9.638950157766153,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": -0.23110198974609375,
+      "logits/rejected": -0.08799050003290176,
+      "logps/chosen": -1.543038010597229,
+      "logps/rejected": -2.047542095184326,
+      "loss": 0.4726,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.543038010597229,
+      "rewards/margins": 0.5045040249824524,
+      "rewards/rejected": -2.047542095184326,
+      "sft_loss": 1.6039879322052002,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 12.648365762769789,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": -0.3018723428249359,
+      "logits/rejected": -0.16592738032341003,
+      "logps/chosen": -1.4970450401306152,
+      "logps/rejected": -2.2717602252960205,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4970450401306152,
+      "rewards/margins": 0.7747151851654053,
+      "rewards/rejected": -2.2717602252960205,
+      "sft_loss": 1.6478235721588135,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 6.599885818260292,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": -0.2461796998977661,
+      "logits/rejected": -0.033960431814193726,
+      "logps/chosen": -1.504396677017212,
+      "logps/rejected": -2.293781280517578,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.504396677017212,
+      "rewards/margins": 0.7893846035003662,
+      "rewards/rejected": -2.293781280517578,
+      "sft_loss": 1.5998690128326416,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 13.023271021475802,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": -0.22546645998954773,
+      "logits/rejected": -0.05487429350614548,
+      "logps/chosen": -1.58822500705719,
+      "logps/rejected": -2.127042293548584,
+      "loss": 0.4672,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.58822500705719,
+      "rewards/margins": 0.5388172268867493,
+      "rewards/rejected": -2.127042293548584,
+      "sft_loss": 1.7143867015838623,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 12.84040076737523,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": -0.24874725937843323,
+      "logits/rejected": -0.10095198452472687,
+      "logps/chosen": -1.6159508228302002,
+      "logps/rejected": -2.2336201667785645,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6159508228302002,
+      "rewards/margins": 0.6176694631576538,
+      "rewards/rejected": -2.2336201667785645,
+      "sft_loss": 1.6980879306793213,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 9.003986896162802,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": -0.2251073122024536,
+      "logits/rejected": -0.04803949221968651,
+      "logps/chosen": -1.5959573984146118,
+      "logps/rejected": -2.325037717819214,
+      "loss": 0.4697,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5959573984146118,
+      "rewards/margins": 0.729080319404602,
+      "rewards/rejected": -2.325037717819214,
+      "sft_loss": 1.7491687536239624,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 10.155414638946343,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": -0.22241003811359406,
+      "logits/rejected": -0.09521248191595078,
+      "logps/chosen": -1.6965595483779907,
+      "logps/rejected": -2.3352200984954834,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6965595483779907,
+      "rewards/margins": 0.6386607885360718,
+      "rewards/rejected": -2.3352200984954834,
+      "sft_loss": 1.7339941263198853,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 6.532500154575078,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": -0.18349921703338623,
+      "logits/rejected": -0.10400259494781494,
+      "logps/chosen": -1.583504557609558,
+      "logps/rejected": -2.2146687507629395,
+      "loss": 0.4526,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.583504557609558,
+      "rewards/margins": 0.6311641335487366,
+      "rewards/rejected": -2.2146687507629395,
+      "sft_loss": 1.6056795120239258,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 9.741375398588128,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": -0.239861398935318,
+      "logits/rejected": -0.054199181497097015,
+      "logps/chosen": -1.4949240684509277,
+      "logps/rejected": -2.2963128089904785,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4949240684509277,
+      "rewards/margins": 0.8013887405395508,
+      "rewards/rejected": -2.2963128089904785,
+      "sft_loss": 1.6196388006210327,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 20.376394353403786,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": -0.28271156549453735,
+      "logits/rejected": -0.20126430690288544,
+      "logps/chosen": -1.7372280359268188,
+      "logps/rejected": -2.233088970184326,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7372280359268188,
+      "rewards/margins": 0.49586066603660583,
+      "rewards/rejected": -2.233088970184326,
+      "sft_loss": 1.8056633472442627,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 8.267825898405084,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": -0.19051404297351837,
+      "logits/rejected": -0.11613442748785019,
+      "logps/chosen": -1.6146233081817627,
+      "logps/rejected": -2.352165699005127,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6146233081817627,
+      "rewards/margins": 0.737542450428009,
+      "rewards/rejected": -2.352165699005127,
+      "sft_loss": 1.7026243209838867,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 6.724335492222933,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": -0.26722902059555054,
+      "logits/rejected": -0.1480477899312973,
+      "logps/chosen": -1.5895460844039917,
+      "logps/rejected": -2.1785476207733154,
+      "loss": 0.4478,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5895460844039917,
+      "rewards/margins": 0.5890012979507446,
+      "rewards/rejected": -2.1785476207733154,
+      "sft_loss": 1.6915168762207031,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 7.925352571352285,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": -0.30871835350990295,
+      "logits/rejected": -0.07422409951686859,
+      "logps/chosen": -1.6400692462921143,
+      "logps/rejected": -2.2754616737365723,
+      "loss": 0.4412,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6400692462921143,
+      "rewards/margins": 0.6353921890258789,
+      "rewards/rejected": -2.2754616737365723,
+      "sft_loss": 1.7322028875350952,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 7.6566753761813615,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": -0.14006388187408447,
+      "logits/rejected": -0.13663342595100403,
+      "logps/chosen": -1.5292609930038452,
+      "logps/rejected": -2.09344220161438,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5292609930038452,
+      "rewards/margins": 0.5641810297966003,
+      "rewards/rejected": -2.09344220161438,
+      "sft_loss": 1.6062122583389282,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 9.318812779298241,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": -0.17240700125694275,
+      "logits/rejected": -0.024434376507997513,
+      "logps/chosen": -1.409942626953125,
+      "logps/rejected": -2.1142661571502686,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.409942626953125,
+      "rewards/margins": 0.7043234705924988,
+      "rewards/rejected": -2.1142661571502686,
+      "sft_loss": 1.484179973602295,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 8.427360444991452,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": -0.30821970105171204,
+      "logits/rejected": -0.09003923833370209,
+      "logps/chosen": -1.610558271408081,
+      "logps/rejected": -2.268453598022461,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.610558271408081,
+      "rewards/margins": 0.6578952074050903,
+      "rewards/rejected": -2.268453598022461,
+      "sft_loss": 1.7595865726470947,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 7.140929143265957,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": -0.26558440923690796,
+      "logits/rejected": -0.05220402404665947,
+      "logps/chosen": -1.6598186492919922,
+      "logps/rejected": -2.2687034606933594,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6598186492919922,
+      "rewards/margins": 0.6088845133781433,
+      "rewards/rejected": -2.2687034606933594,
+      "sft_loss": 1.7320663928985596,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 13.276671883952789,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": -0.3073621690273285,
+      "logits/rejected": -0.14894798398017883,
+      "logps/chosen": -1.6358146667480469,
+      "logps/rejected": -2.2647910118103027,
+      "loss": 0.4647,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6358146667480469,
+      "rewards/margins": 0.6289765238761902,
+      "rewards/rejected": -2.2647910118103027,
+      "sft_loss": 1.7382218837738037,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 5.620089072897995,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": -0.25026455521583557,
+      "logits/rejected": -0.10177797079086304,
+      "logps/chosen": -1.5480822324752808,
+      "logps/rejected": -2.1520581245422363,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5480822324752808,
+      "rewards/margins": 0.6039758920669556,
+      "rewards/rejected": -2.1520581245422363,
+      "sft_loss": 1.6605409383773804,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 8.59565820868914,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": -0.18619614839553833,
+      "logits/rejected": -0.018567675724625587,
+      "logps/chosen": -1.5407416820526123,
+      "logps/rejected": -2.2864482402801514,
+      "loss": 0.4455,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5407416820526123,
+      "rewards/margins": 0.7457064986228943,
+      "rewards/rejected": -2.2864482402801514,
+      "sft_loss": 1.6421838998794556,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 6.951569508645217,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": -0.2967742383480072,
+      "logits/rejected": -0.1319170445203781,
+      "logps/chosen": -1.5985156297683716,
+      "logps/rejected": -2.249882221221924,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5985156297683716,
+      "rewards/margins": 0.651366651058197,
+      "rewards/rejected": -2.249882221221924,
+      "sft_loss": 1.6927413940429688,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": 0.1534985452890396,
+      "eval_logits/rejected": 0.2581808865070343,
+      "eval_logps/chosen": -1.6394926309585571,
+      "eval_logps/rejected": -2.227719783782959,
+      "eval_loss": 0.49088484048843384,
+      "eval_rewards/accuracies": 0.6735904812812805,
+      "eval_rewards/chosen": -1.6394926309585571,
+      "eval_rewards/margins": 0.5882269144058228,
+      "eval_rewards/rejected": -2.227719783782959,
+      "eval_runtime": 43.4714,
+      "eval_samples_per_second": 30.94,
+      "eval_sft_loss": 1.7121342420578003,
+      "eval_steps_per_second": 7.752,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 8.04651481113309,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": -0.28536808490753174,
+      "logits/rejected": -0.24391081929206848,
+      "logps/chosen": -1.5283732414245605,
+      "logps/rejected": -2.0771474838256836,
+      "loss": 0.4629,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5283732414245605,
+      "rewards/margins": 0.548774242401123,
+      "rewards/rejected": -2.0771474838256836,
+      "sft_loss": 1.613776445388794,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 7.95291662888706,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": -0.23777012526988983,
+      "logits/rejected": -0.1785149872303009,
+      "logps/chosen": -1.6985456943511963,
+      "logps/rejected": -2.46702241897583,
+      "loss": 0.4479,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6985456943511963,
+      "rewards/margins": 0.7684768438339233,
+      "rewards/rejected": -2.46702241897583,
+      "sft_loss": 1.7738962173461914,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 7.746688363671087,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": -0.23160281777381897,
+      "logits/rejected": -0.04335252568125725,
+      "logps/chosen": -1.609033226966858,
+      "logps/rejected": -2.23053240776062,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.609033226966858,
+      "rewards/margins": 0.6214991807937622,
+      "rewards/rejected": -2.23053240776062,
+      "sft_loss": 1.676290512084961,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 10.610213280300115,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": -0.2662007212638855,
+      "logits/rejected": -0.17510481178760529,
+      "logps/chosen": -1.6208078861236572,
+      "logps/rejected": -2.1000378131866455,
+      "loss": 0.4793,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6208078861236572,
+      "rewards/margins": 0.4792299270629883,
+      "rewards/rejected": -2.1000378131866455,
+      "sft_loss": 1.692181944847107,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 8.936766324818056,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": -0.23490257561206818,
+      "logits/rejected": -0.11895327270030975,
+      "logps/chosen": -1.4317034482955933,
+      "logps/rejected": -2.0047073364257812,
+      "loss": 0.4219,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4317034482955933,
+      "rewards/margins": 0.573003888130188,
+      "rewards/rejected": -2.0047073364257812,
+      "sft_loss": 1.589618444442749,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 7.21487936412045,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": -0.25477251410484314,
+      "logits/rejected": -0.06471340358257294,
+      "logps/chosen": -1.6068341732025146,
+      "logps/rejected": -2.2490625381469727,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6068341732025146,
+      "rewards/margins": 0.6422284841537476,
+      "rewards/rejected": -2.2490625381469727,
+      "sft_loss": 1.6686866283416748,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 6.72950333850293,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": -0.32577234506607056,
+      "logits/rejected": -0.0894775241613388,
+      "logps/chosen": -1.5162084102630615,
+      "logps/rejected": -2.182002544403076,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5162084102630615,
+      "rewards/margins": 0.6657941341400146,
+      "rewards/rejected": -2.182002544403076,
+      "sft_loss": 1.6630744934082031,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 9.555770264870109,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": -0.33920782804489136,
+      "logits/rejected": -0.09382447600364685,
+      "logps/chosen": -1.5818603038787842,
+      "logps/rejected": -2.1809945106506348,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5818603038787842,
+      "rewards/margins": 0.5991338491439819,
+      "rewards/rejected": -2.1809945106506348,
+      "sft_loss": 1.6767442226409912,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 8.69159399901978,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": -0.2876212000846863,
+      "logits/rejected": -0.08851093053817749,
+      "logps/chosen": -1.5860512256622314,
+      "logps/rejected": -2.099834442138672,
+      "loss": 0.4902,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5860512256622314,
+      "rewards/margins": 0.5137835144996643,
+      "rewards/rejected": -2.099834442138672,
+      "sft_loss": 1.675398588180542,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 13.63365552934808,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": -0.23422542214393616,
+      "logits/rejected": -0.09559222310781479,
+      "logps/chosen": -1.6235287189483643,
+      "logps/rejected": -2.3158679008483887,
+      "loss": 0.4842,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6235287189483643,
+      "rewards/margins": 0.692339301109314,
+      "rewards/rejected": -2.3158679008483887,
+      "sft_loss": 1.6855251789093018,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 12.73541327236651,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": -0.2981303334236145,
+      "logits/rejected": -0.18724291026592255,
+      "logps/chosen": -1.5792548656463623,
+      "logps/rejected": -2.230262517929077,
+      "loss": 0.4532,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5792548656463623,
+      "rewards/margins": 0.6510077118873596,
+      "rewards/rejected": -2.230262517929077,
+      "sft_loss": 1.7070000171661377,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 9.25717711299543,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": -0.33461794257164,
+      "logits/rejected": -0.09690986573696136,
+      "logps/chosen": -1.5364824533462524,
+      "logps/rejected": -2.3159992694854736,
+      "loss": 0.4211,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5364824533462524,
+      "rewards/margins": 0.7795166969299316,
+      "rewards/rejected": -2.3159992694854736,
+      "sft_loss": 1.691641092300415,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 11.18028000847737,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": -0.3134737014770508,
+      "logits/rejected": -0.10241047292947769,
+      "logps/chosen": -1.6970819234848022,
+      "logps/rejected": -2.298682451248169,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6970819234848022,
+      "rewards/margins": 0.6016003489494324,
+      "rewards/rejected": -2.298682451248169,
+      "sft_loss": 1.7689815759658813,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 7.14924392499626,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": -0.14032816886901855,
+      "logits/rejected": -0.053854428231716156,
+      "logps/chosen": -1.4425265789031982,
+      "logps/rejected": -2.010845184326172,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4425265789031982,
+      "rewards/margins": 0.568318784236908,
+      "rewards/rejected": -2.010845184326172,
+      "sft_loss": 1.5330469608306885,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 14.295657293369166,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": -0.24750380218029022,
+      "logits/rejected": -0.003714990569278598,
+      "logps/chosen": -1.5968362092971802,
+      "logps/rejected": -2.2002665996551514,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5968362092971802,
+      "rewards/margins": 0.6034305691719055,
+      "rewards/rejected": -2.2002665996551514,
+      "sft_loss": 1.6557786464691162,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 13.346358629047302,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": -0.24167075753211975,
+      "logits/rejected": -0.14121203124523163,
+      "logps/chosen": -1.4179035425186157,
+      "logps/rejected": -1.9109907150268555,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4179035425186157,
+      "rewards/margins": 0.49308720231056213,
+      "rewards/rejected": -1.9109907150268555,
+      "sft_loss": 1.5340925455093384,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 8.027036857562305,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": -0.21096546947956085,
+      "logits/rejected": -0.11151149123907089,
+      "logps/chosen": -1.5817596912384033,
+      "logps/rejected": -2.2323250770568848,
+      "loss": 0.4462,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5817596912384033,
+      "rewards/margins": 0.650565505027771,
+      "rewards/rejected": -2.2323250770568848,
+      "sft_loss": 1.6181080341339111,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 9.416302960957523,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": -0.2919323444366455,
+      "logits/rejected": -0.05769073963165283,
+      "logps/chosen": -1.5122710466384888,
+      "logps/rejected": -2.2569687366485596,
+      "loss": 0.4156,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5122710466384888,
+      "rewards/margins": 0.7446975111961365,
+      "rewards/rejected": -2.2569687366485596,
+      "sft_loss": 1.627769112586975,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 12.782013253866758,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": -0.33929866552352905,
+      "logits/rejected": -0.11169986426830292,
+      "logps/chosen": -1.5895986557006836,
+      "logps/rejected": -2.131399631500244,
+      "loss": 0.4571,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5895986557006836,
+      "rewards/margins": 0.5418012142181396,
+      "rewards/rejected": -2.131399631500244,
+      "sft_loss": 1.735253930091858,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 9.718553496516888,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": -0.15040044486522675,
+      "logits/rejected": -0.14154385030269623,
+      "logps/chosen": -1.5411628484725952,
+      "logps/rejected": -2.1179873943328857,
+      "loss": 0.4453,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5411628484725952,
+      "rewards/margins": 0.5768247246742249,
+      "rewards/rejected": -2.1179873943328857,
+      "sft_loss": 1.6429777145385742,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 6.6985265123131486,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": -0.2672615945339203,
+      "logits/rejected": -0.09151863306760788,
+      "logps/chosen": -1.5106264352798462,
+      "logps/rejected": -2.1163084506988525,
+      "loss": 0.4391,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5106264352798462,
+      "rewards/margins": 0.6056820154190063,
+      "rewards/rejected": -2.1163084506988525,
+      "sft_loss": 1.5961272716522217,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 7.265679258943073,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": -0.2920701205730438,
+      "logits/rejected": -0.13715091347694397,
+      "logps/chosen": -1.5710780620574951,
+      "logps/rejected": -2.125671863555908,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5710780620574951,
+      "rewards/margins": 0.5545935034751892,
+      "rewards/rejected": -2.125671863555908,
+      "sft_loss": 1.6058059930801392,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 7.190580914210609,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": -0.2794167399406433,
+      "logits/rejected": -0.1814873069524765,
+      "logps/chosen": -1.5710397958755493,
+      "logps/rejected": -2.111194372177124,
+      "loss": 0.4604,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5710397958755493,
+      "rewards/margins": 0.540154755115509,
+      "rewards/rejected": -2.111194372177124,
+      "sft_loss": 1.6599490642547607,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 10.670685800851055,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": -0.22668643295764923,
+      "logits/rejected": -0.11982636153697968,
+      "logps/chosen": -1.5077179670333862,
+      "logps/rejected": -2.278191328048706,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5077179670333862,
+      "rewards/margins": 0.7704734802246094,
+      "rewards/rejected": -2.278191328048706,
+      "sft_loss": 1.5749647617340088,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 9.584274557121692,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": -0.33840879797935486,
+      "logits/rejected": -0.14164835214614868,
+      "logps/chosen": -1.6478707790374756,
+      "logps/rejected": -2.3922953605651855,
+      "loss": 0.454,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6478707790374756,
+      "rewards/margins": 0.7444245219230652,
+      "rewards/rejected": -2.3922953605651855,
+      "sft_loss": 1.7673391103744507,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 9.0319511589624,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": -0.27057284116744995,
+      "logits/rejected": -0.16721342504024506,
+      "logps/chosen": -1.5868384838104248,
+      "logps/rejected": -2.053842544555664,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5868384838104248,
+      "rewards/margins": 0.4670039713382721,
+      "rewards/rejected": -2.053842544555664,
+      "sft_loss": 1.639451026916504,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 10.014783939770096,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": -0.13392801582813263,
+      "logits/rejected": -0.009936687536537647,
+      "logps/chosen": -1.5058220624923706,
+      "logps/rejected": -2.101733446121216,
+      "loss": 0.463,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5058220624923706,
+      "rewards/margins": 0.5959112644195557,
+      "rewards/rejected": -2.101733446121216,
+      "sft_loss": 1.6472076177597046,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 5.789686194060652,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": -0.1509685218334198,
+      "logits/rejected": -0.054625581949949265,
+      "logps/chosen": -1.4414759874343872,
+      "logps/rejected": -2.034996747970581,
+      "loss": 0.4576,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4414759874343872,
+      "rewards/margins": 0.5935209393501282,
+      "rewards/rejected": -2.034996747970581,
+      "sft_loss": 1.5240309238433838,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 8.089792084934379,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": -0.1864078938961029,
+      "logits/rejected": -0.0398380383849144,
+      "logps/chosen": -1.5171597003936768,
+      "logps/rejected": -2.1239724159240723,
+      "loss": 0.4606,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5171597003936768,
+      "rewards/margins": 0.6068126559257507,
+      "rewards/rejected": -2.1239724159240723,
+      "sft_loss": 1.605342149734497,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 7.886331372210549,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": -0.28180861473083496,
+      "logits/rejected": -0.1611512005329132,
+      "logps/chosen": -1.4591307640075684,
+      "logps/rejected": -2.0682687759399414,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4591307640075684,
+      "rewards/margins": 0.6091380715370178,
+      "rewards/rejected": -2.0682687759399414,
+      "sft_loss": 1.5430991649627686,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 7.17269276476902,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": -0.21525149047374725,
+      "logits/rejected": -0.13745595514774323,
+      "logps/chosen": -1.6477893590927124,
+      "logps/rejected": -2.127769708633423,
+      "loss": 0.479,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6477893590927124,
+      "rewards/margins": 0.47998008131980896,
+      "rewards/rejected": -2.127769708633423,
+      "sft_loss": 1.786193609237671,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 6.662430669419078,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": -0.28460606932640076,
+      "logits/rejected": -0.08241887390613556,
+      "logps/chosen": -1.5029640197753906,
+      "logps/rejected": -2.1092498302459717,
+      "loss": 0.49,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5029640197753906,
+      "rewards/margins": 0.6062856912612915,
+      "rewards/rejected": -2.1092498302459717,
+      "sft_loss": 1.5308010578155518,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 8.137985723376088,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": -0.207010418176651,
+      "logits/rejected": -0.09890707582235336,
+      "logps/chosen": -1.6632816791534424,
+      "logps/rejected": -2.2407917976379395,
+      "loss": 0.4693,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6632816791534424,
+      "rewards/margins": 0.5775102376937866,
+      "rewards/rejected": -2.2407917976379395,
+      "sft_loss": 1.731000304222107,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 13.854076075939032,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": -0.15999329090118408,
+      "logits/rejected": 0.00819392316043377,
+      "logps/chosen": -1.517242431640625,
+      "logps/rejected": -2.212289333343506,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.517242431640625,
+      "rewards/margins": 0.6950467228889465,
+      "rewards/rejected": -2.212289333343506,
+      "sft_loss": 1.5932613611221313,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 11.39520887739819,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": -0.1302533894777298,
+      "logits/rejected": -0.06674132496118546,
+      "logps/chosen": -1.5052393674850464,
+      "logps/rejected": -2.0828909873962402,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5052393674850464,
+      "rewards/margins": 0.5776516199111938,
+      "rewards/rejected": -2.0828909873962402,
+      "sft_loss": 1.5766162872314453,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 7.304941995094994,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": -0.200159952044487,
+      "logits/rejected": -0.14625905454158783,
+      "logps/chosen": -1.4850728511810303,
+      "logps/rejected": -2.0518252849578857,
+      "loss": 0.4484,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4850728511810303,
+      "rewards/margins": 0.566752552986145,
+      "rewards/rejected": -2.0518252849578857,
+      "sft_loss": 1.6040903329849243,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 8.850165611803599,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": -0.2055172473192215,
+      "logits/rejected": -0.04428768530488014,
+      "logps/chosen": -1.6074609756469727,
+      "logps/rejected": -2.1912786960601807,
+      "loss": 0.5067,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6074609756469727,
+      "rewards/margins": 0.5838177800178528,
+      "rewards/rejected": -2.1912786960601807,
+      "sft_loss": 1.7369712591171265,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 6.256203184747748,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": -0.3060024380683899,
+      "logits/rejected": -0.11156409978866577,
+      "logps/chosen": -1.4236325025558472,
+      "logps/rejected": -1.9613687992095947,
+      "loss": 0.4619,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4236325025558472,
+      "rewards/margins": 0.5377359986305237,
+      "rewards/rejected": -1.9613687992095947,
+      "sft_loss": 1.5441299676895142,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 12.088895012902688,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": -0.25038835406303406,
+      "logits/rejected": -0.11709809303283691,
+      "logps/chosen": -1.5372766256332397,
+      "logps/rejected": -2.185891628265381,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5372766256332397,
+      "rewards/margins": 0.6486150026321411,
+      "rewards/rejected": -2.185891628265381,
+      "sft_loss": 1.6226387023925781,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 8.580375617672612,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": -0.2508789002895355,
+      "logits/rejected": 0.03930927440524101,
+      "logps/chosen": -1.5669784545898438,
+      "logps/rejected": -2.2951834201812744,
+      "loss": 0.4218,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5669784545898438,
+      "rewards/margins": 0.7282050848007202,
+      "rewards/rejected": -2.2951834201812744,
+      "sft_loss": 1.6732451915740967,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 7.851497136140735,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": -0.20749421417713165,
+      "logits/rejected": -0.039168864488601685,
+      "logps/chosen": -1.5519232749938965,
+      "logps/rejected": -2.2856459617614746,
+      "loss": 0.4607,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5519232749938965,
+      "rewards/margins": 0.7337230443954468,
+      "rewards/rejected": -2.2856459617614746,
+      "sft_loss": 1.6616754531860352,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 10.618025974762887,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": -0.2528127431869507,
+      "logits/rejected": -0.09626563638448715,
+      "logps/chosen": -1.6161024570465088,
+      "logps/rejected": -2.264498710632324,
+      "loss": 0.4753,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6161024570465088,
+      "rewards/margins": 0.6483960747718811,
+      "rewards/rejected": -2.264498710632324,
+      "sft_loss": 1.6971594095230103,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 10.594551544596676,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": -0.18787157535552979,
+      "logits/rejected": -0.026337753981351852,
+      "logps/chosen": -1.6552670001983643,
+      "logps/rejected": -2.4452872276306152,
+      "loss": 0.4485,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6552670001983643,
+      "rewards/margins": 0.7900202870368958,
+      "rewards/rejected": -2.4452872276306152,
+      "sft_loss": 1.7910692691802979,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 7.410887227720804,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": -0.20163783431053162,
+      "logits/rejected": 0.02683967910706997,
+      "logps/chosen": -1.464026689529419,
+      "logps/rejected": -2.135082721710205,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.464026689529419,
+      "rewards/margins": 0.67105633020401,
+      "rewards/rejected": -2.135082721710205,
+      "sft_loss": 1.6336700916290283,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 8.391502482850902,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": -0.32311874628067017,
+      "logits/rejected": -0.2665711045265198,
+      "logps/chosen": -1.5374579429626465,
+      "logps/rejected": -2.08915376663208,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5374579429626465,
+      "rewards/margins": 0.5516960024833679,
+      "rewards/rejected": -2.08915376663208,
+      "sft_loss": 1.6142950057983398,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 8.433282652426243,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": -0.1706085354089737,
+      "logits/rejected": -0.07626998424530029,
+      "logps/chosen": -1.5323286056518555,
+      "logps/rejected": -2.2087883949279785,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5323286056518555,
+      "rewards/margins": 0.6764597296714783,
+      "rewards/rejected": -2.2087883949279785,
+      "sft_loss": 1.691072702407837,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 10.54087091911546,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": -0.14578501880168915,
+      "logits/rejected": 0.02640867792069912,
+      "logps/chosen": -1.5427324771881104,
+      "logps/rejected": -2.2831692695617676,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5427324771881104,
+      "rewards/margins": 0.7404367923736572,
+      "rewards/rejected": -2.2831692695617676,
+      "sft_loss": 1.5968652963638306,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 6.8818274016401695,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": -0.16421422362327576,
+      "logits/rejected": -0.039952315390110016,
+      "logps/chosen": -1.5872137546539307,
+      "logps/rejected": -2.258237838745117,
+      "loss": 0.4139,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5872137546539307,
+      "rewards/margins": 0.6710240244865417,
+      "rewards/rejected": -2.258237838745117,
+      "sft_loss": 1.6829907894134521,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 7.498430334973799,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": -0.20430341362953186,
+      "logits/rejected": -0.04681653156876564,
+      "logps/chosen": -1.5307071208953857,
+      "logps/rejected": -2.1223111152648926,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5307071208953857,
+      "rewards/margins": 0.5916039347648621,
+      "rewards/rejected": -2.1223111152648926,
+      "sft_loss": 1.6639989614486694,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 8.142152141956702,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": -0.2613300681114197,
+      "logits/rejected": 0.011642465367913246,
+      "logps/chosen": -1.5273935794830322,
+      "logps/rejected": -2.2350680828094482,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5273935794830322,
+      "rewards/margins": 0.7076746225357056,
+      "rewards/rejected": -2.2350680828094482,
+      "sft_loss": 1.5699331760406494,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 11.092590122881404,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": -0.2698579430580139,
+      "logits/rejected": -0.10053672641515732,
+      "logps/chosen": -1.6180152893066406,
+      "logps/rejected": -2.2728400230407715,
+      "loss": 0.4355,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6180152893066406,
+      "rewards/margins": 0.6548250317573547,
+      "rewards/rejected": -2.2728400230407715,
+      "sft_loss": 1.798405408859253,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 10.070924877852471,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": -0.26223617792129517,
+      "logits/rejected": -0.1275007724761963,
+      "logps/chosen": -1.6201118230819702,
+      "logps/rejected": -2.2663655281066895,
+      "loss": 0.4412,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6201118230819702,
+      "rewards/margins": 0.6462540626525879,
+      "rewards/rejected": -2.2663655281066895,
+      "sft_loss": 1.7024576663970947,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 9.141394981691903,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": -0.21037046611309052,
+      "logits/rejected": -0.12881790101528168,
+      "logps/chosen": -1.6491973400115967,
+      "logps/rejected": -2.169259548187256,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6491973400115967,
+      "rewards/margins": 0.5200623273849487,
+      "rewards/rejected": -2.169259548187256,
+      "sft_loss": 1.6937086582183838,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 15.363046070698873,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": -0.24200613796710968,
+      "logits/rejected": -0.09714044630527496,
+      "logps/chosen": -1.562156319618225,
+      "logps/rejected": -2.008108615875244,
+      "loss": 0.48,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.562156319618225,
+      "rewards/margins": 0.4459524154663086,
+      "rewards/rejected": -2.008108615875244,
+      "sft_loss": 1.6635644435882568,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 13.921499153222394,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": -0.163819819688797,
+      "logits/rejected": 0.010681016370654106,
+      "logps/chosen": -1.5085991621017456,
+      "logps/rejected": -2.1517558097839355,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5085991621017456,
+      "rewards/margins": 0.6431564092636108,
+      "rewards/rejected": -2.1517558097839355,
+      "sft_loss": 1.5810153484344482,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 13.998491599616301,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": -0.15430161356925964,
+      "logits/rejected": -0.03651545196771622,
+      "logps/chosen": -1.5104106664657593,
+      "logps/rejected": -2.1672616004943848,
+      "loss": 0.454,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5104106664657593,
+      "rewards/margins": 0.6568506956100464,
+      "rewards/rejected": -2.1672616004943848,
+      "sft_loss": 1.551937460899353,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 9.441523703284677,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": -0.29489654302597046,
+      "logits/rejected": -0.07090181112289429,
+      "logps/chosen": -1.531465768814087,
+      "logps/rejected": -2.1749260425567627,
+      "loss": 0.4285,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.531465768814087,
+      "rewards/margins": 0.6434603333473206,
+      "rewards/rejected": -2.1749260425567627,
+      "sft_loss": 1.6144222021102905,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 8.926537151475863,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": -0.175274059176445,
+      "logits/rejected": -0.11035673320293427,
+      "logps/chosen": -1.5257591009140015,
+      "logps/rejected": -2.1212382316589355,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5257591009140015,
+      "rewards/margins": 0.5954791307449341,
+      "rewards/rejected": -2.1212382316589355,
+      "sft_loss": 1.582589864730835,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 6.535008558907057,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": -0.20110678672790527,
+      "logits/rejected": 0.01152988150715828,
+      "logps/chosen": -1.5889029502868652,
+      "logps/rejected": -2.2240920066833496,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5889029502868652,
+      "rewards/margins": 0.6351887583732605,
+      "rewards/rejected": -2.2240920066833496,
+      "sft_loss": 1.6570899486541748,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 9.172661414479018,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": -0.2919153571128845,
+      "logits/rejected": -0.1516752541065216,
+      "logps/chosen": -1.5378618240356445,
+      "logps/rejected": -2.1191885471343994,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5378618240356445,
+      "rewards/margins": 0.5813267827033997,
+      "rewards/rejected": -2.1191885471343994,
+      "sft_loss": 1.6722614765167236,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 10.146117393815855,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": -0.16152144968509674,
+      "logits/rejected": -0.10950030386447906,
+      "logps/chosen": -1.5789520740509033,
+      "logps/rejected": -2.131608486175537,
+      "loss": 0.484,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5789520740509033,
+      "rewards/margins": 0.552656352519989,
+      "rewards/rejected": -2.131608486175537,
+      "sft_loss": 1.6074949502944946,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 9.29795942326603,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": -0.27605026960372925,
+      "logits/rejected": -0.09732412546873093,
+      "logps/chosen": -1.629088044166565,
+      "logps/rejected": -2.297694683074951,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.629088044166565,
+      "rewards/margins": 0.668606698513031,
+      "rewards/rejected": -2.297694683074951,
+      "sft_loss": 1.7505499124526978,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 7.771825923021724,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": -0.2481086701154709,
+      "logits/rejected": -0.19722998142242432,
+      "logps/chosen": -1.6821883916854858,
+      "logps/rejected": -2.349914789199829,
+      "loss": 0.4477,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6821883916854858,
+      "rewards/margins": 0.6677263379096985,
+      "rewards/rejected": -2.349914789199829,
+      "sft_loss": 1.7744452953338623,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 8.11669941970569,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": -0.2124921828508377,
+      "logits/rejected": -0.09005747735500336,
+      "logps/chosen": -1.4214861392974854,
+      "logps/rejected": -2.079423427581787,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4214861392974854,
+      "rewards/margins": 0.6579370498657227,
+      "rewards/rejected": -2.079423427581787,
+      "sft_loss": 1.522826910018921,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 9.06402656599577,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": -0.2556040287017822,
+      "logits/rejected": -0.1857677698135376,
+      "logps/chosen": -1.5797942876815796,
+      "logps/rejected": -2.15522837638855,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5797942876815796,
+      "rewards/margins": 0.5754343867301941,
+      "rewards/rejected": -2.15522837638855,
+      "sft_loss": 1.672151803970337,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 6.210909136154958,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": -0.22033897042274475,
+      "logits/rejected": -0.08500131219625473,
+      "logps/chosen": -1.5027967691421509,
+      "logps/rejected": -2.132045269012451,
+      "loss": 0.4385,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5027967691421509,
+      "rewards/margins": 0.6292486786842346,
+      "rewards/rejected": -2.132045269012451,
+      "sft_loss": 1.6667730808258057,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 9.608845457903653,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": -0.1915491223335266,
+      "logits/rejected": -0.24349656701087952,
+      "logps/chosen": -1.6221160888671875,
+      "logps/rejected": -2.2130630016326904,
+      "loss": 0.4397,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6221160888671875,
+      "rewards/margins": 0.5909469723701477,
+      "rewards/rejected": -2.2130630016326904,
+      "sft_loss": 1.698195457458496,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 6.093434532048691,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": -0.288370281457901,
+      "logits/rejected": -0.13510087132453918,
+      "logps/chosen": -1.646715521812439,
+      "logps/rejected": -2.3017725944519043,
+      "loss": 0.4489,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.646715521812439,
+      "rewards/margins": 0.6550571322441101,
+      "rewards/rejected": -2.3017725944519043,
+      "sft_loss": 1.7606605291366577,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 10.204274927929957,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": -0.22085830569267273,
+      "logits/rejected": -0.16141316294670105,
+      "logps/chosen": -1.7018508911132812,
+      "logps/rejected": -2.2902958393096924,
+      "loss": 0.4871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7018508911132812,
+      "rewards/margins": 0.5884448289871216,
+      "rewards/rejected": -2.2902958393096924,
+      "sft_loss": 1.768785834312439,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 5.708839072897948,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": -0.18627746403217316,
+      "logits/rejected": 0.04857007414102554,
+      "logps/chosen": -1.5648773908615112,
+      "logps/rejected": -2.46248197555542,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5648773908615112,
+      "rewards/margins": 0.8976048231124878,
+      "rewards/rejected": -2.46248197555542,
+      "sft_loss": 1.6237529516220093,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 10.876639185853666,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": -0.2526320815086365,
+      "logits/rejected": -0.06511383503675461,
+      "logps/chosen": -1.4771053791046143,
+      "logps/rejected": -2.261756181716919,
+      "loss": 0.429,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4771053791046143,
+      "rewards/margins": 0.7846506834030151,
+      "rewards/rejected": -2.261756181716919,
+      "sft_loss": 1.570378065109253,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 6.0257970161467425,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": -0.3068903982639313,
+      "logits/rejected": -0.14332722127437592,
+      "logps/chosen": -1.4803614616394043,
+      "logps/rejected": -2.126192331314087,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4803614616394043,
+      "rewards/margins": 0.6458306312561035,
+      "rewards/rejected": -2.126192331314087,
+      "sft_loss": 1.6149470806121826,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 8.333009520000408,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": -0.28441572189331055,
+      "logits/rejected": -0.09357120841741562,
+      "logps/chosen": -1.544640064239502,
+      "logps/rejected": -2.028838634490967,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.544640064239502,
+      "rewards/margins": 0.484198659658432,
+      "rewards/rejected": -2.028838634490967,
+      "sft_loss": 1.6012932062149048,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 10.247917839602298,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": -0.24177256226539612,
+      "logits/rejected": -0.11481799930334091,
+      "logps/chosen": -1.6507999897003174,
+      "logps/rejected": -2.315504789352417,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6507999897003174,
+      "rewards/margins": 0.6647049188613892,
+      "rewards/rejected": -2.315504789352417,
+      "sft_loss": 1.7305183410644531,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 8.516493509100233,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": -0.11893805116415024,
+      "logits/rejected": 0.0038652196526527405,
+      "logps/chosen": -1.5405712127685547,
+      "logps/rejected": -2.211561918258667,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5405712127685547,
+      "rewards/margins": 0.6709908246994019,
+      "rewards/rejected": -2.211561918258667,
+      "sft_loss": 1.5511837005615234,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 8.246441974893472,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": -0.21815164387226105,
+      "logits/rejected": -0.10540630668401718,
+      "logps/chosen": -1.6007814407348633,
+      "logps/rejected": -2.1758387088775635,
+      "loss": 0.4594,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6007814407348633,
+      "rewards/margins": 0.5750571489334106,
+      "rewards/rejected": -2.1758387088775635,
+      "sft_loss": 1.6869404315948486,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 7.714815198032542,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": -0.22387388348579407,
+      "logits/rejected": -0.15563735365867615,
+      "logps/chosen": -1.524994134902954,
+      "logps/rejected": -2.0110983848571777,
+      "loss": 0.4758,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.524994134902954,
+      "rewards/margins": 0.48610442876815796,
+      "rewards/rejected": -2.0110983848571777,
+      "sft_loss": 1.604852318763733,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 6.979427476279799,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": -0.314725786447525,
+      "logits/rejected": -0.16046643257141113,
+      "logps/chosen": -1.570236325263977,
+      "logps/rejected": -2.2742481231689453,
+      "loss": 0.4525,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.570236325263977,
+      "rewards/margins": 0.7040120363235474,
+      "rewards/rejected": -2.2742481231689453,
+      "sft_loss": 1.6277891397476196,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 14.03904377279677,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": -0.20811104774475098,
+      "logits/rejected": -0.11981604248285294,
+      "logps/chosen": -1.591090440750122,
+      "logps/rejected": -2.183060646057129,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.591090440750122,
+      "rewards/margins": 0.5919705033302307,
+      "rewards/rejected": -2.183060646057129,
+      "sft_loss": 1.6530237197875977,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 6.205669366490727,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": -0.23087351024150848,
+      "logits/rejected": -0.10391799360513687,
+      "logps/chosen": -1.468090295791626,
+      "logps/rejected": -2.0137531757354736,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.468090295791626,
+      "rewards/margins": 0.5456628799438477,
+      "rewards/rejected": -2.0137531757354736,
+      "sft_loss": 1.6055347919464111,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": 0.13403891026973724,
+      "eval_logits/rejected": 0.23705637454986572,
+      "eval_logps/chosen": -1.615133285522461,
+      "eval_logps/rejected": -2.1880459785461426,
+      "eval_loss": 0.489407479763031,
+      "eval_rewards/accuracies": 0.669881284236908,
+      "eval_rewards/chosen": -1.615133285522461,
+      "eval_rewards/margins": 0.5729125738143921,
+      "eval_rewards/rejected": -2.1880459785461426,
+      "eval_runtime": 43.3666,
+      "eval_samples_per_second": 31.015,
+      "eval_sft_loss": 1.6890332698822021,
+      "eval_steps_per_second": 7.771,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 9.126873683344243,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": -0.31909486651420593,
+      "logits/rejected": -0.18140074610710144,
+      "logps/chosen": -1.5817447900772095,
+      "logps/rejected": -2.2661678791046143,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5817447900772095,
+      "rewards/margins": 0.6844232678413391,
+      "rewards/rejected": -2.2661678791046143,
+      "sft_loss": 1.6768858432769775,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 9.655498565639416,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": -0.21249334514141083,
+      "logits/rejected": -0.06652887165546417,
+      "logps/chosen": -1.644213318824768,
+      "logps/rejected": -2.251002788543701,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.644213318824768,
+      "rewards/margins": 0.6067894697189331,
+      "rewards/rejected": -2.251002788543701,
+      "sft_loss": 1.711158037185669,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 7.29758415690169,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": -0.19294914603233337,
+      "logits/rejected": -0.05259813740849495,
+      "logps/chosen": -1.3905773162841797,
+      "logps/rejected": -2.1160082817077637,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3905773162841797,
+      "rewards/margins": 0.725430965423584,
+      "rewards/rejected": -2.1160082817077637,
+      "sft_loss": 1.5208027362823486,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 8.18138114361707,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": -0.16653354465961456,
+      "logits/rejected": -0.0694139152765274,
+      "logps/chosen": -1.596074104309082,
+      "logps/rejected": -2.121253252029419,
+      "loss": 0.4828,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.596074104309082,
+      "rewards/margins": 0.5251791477203369,
+      "rewards/rejected": -2.121253252029419,
+      "sft_loss": 1.616479516029358,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 12.856515905683183,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": -0.19333486258983612,
+      "logits/rejected": -0.05315401405096054,
+      "logps/chosen": -1.5848066806793213,
+      "logps/rejected": -2.146439790725708,
+      "loss": 0.4625,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5848066806793213,
+      "rewards/margins": 0.5616329908370972,
+      "rewards/rejected": -2.146439790725708,
+      "sft_loss": 1.6240943670272827,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 12.89972314598527,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": -0.2448471337556839,
+      "logits/rejected": -0.07540064305067062,
+      "logps/chosen": -1.600976586341858,
+      "logps/rejected": -2.113248109817505,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.600976586341858,
+      "rewards/margins": 0.5122717022895813,
+      "rewards/rejected": -2.113248109817505,
+      "sft_loss": 1.6421327590942383,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 8.822778276424396,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": -0.35598570108413696,
+      "logits/rejected": -0.14402401447296143,
+      "logps/chosen": -1.546924352645874,
+      "logps/rejected": -2.2639241218566895,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.546924352645874,
+      "rewards/margins": 0.7169994711875916,
+      "rewards/rejected": -2.2639241218566895,
+      "sft_loss": 1.6327308416366577,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 9.760793670851609,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": -0.2984338700771332,
+      "logits/rejected": -0.0784219279885292,
+      "logps/chosen": -1.5313817262649536,
+      "logps/rejected": -2.211430072784424,
+      "loss": 0.4308,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5313817262649536,
+      "rewards/margins": 0.6800485849380493,
+      "rewards/rejected": -2.211430072784424,
+      "sft_loss": 1.6355266571044922,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 7.753373059889907,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": -0.24122004210948944,
+      "logits/rejected": -0.13646744191646576,
+      "logps/chosen": -1.5357145071029663,
+      "logps/rejected": -2.123054265975952,
+      "loss": 0.4625,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5357145071029663,
+      "rewards/margins": 0.5873396992683411,
+      "rewards/rejected": -2.123054265975952,
+      "sft_loss": 1.6655117273330688,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 10.372174907711964,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": -0.38891011476516724,
+      "logits/rejected": -0.22265692055225372,
+      "logps/chosen": -1.5839711427688599,
+      "logps/rejected": -2.3328089714050293,
+      "loss": 0.4537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5839711427688599,
+      "rewards/margins": 0.748837947845459,
+      "rewards/rejected": -2.3328089714050293,
+      "sft_loss": 1.6295592784881592,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 9.388446175901082,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": -0.21261592209339142,
+      "logits/rejected": -0.046322330832481384,
+      "logps/chosen": -1.467795729637146,
+      "logps/rejected": -2.14953351020813,
+      "loss": 0.4437,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.467795729637146,
+      "rewards/margins": 0.6817376017570496,
+      "rewards/rejected": -2.14953351020813,
+      "sft_loss": 1.5824216604232788,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 9.932866439440142,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": -0.21794529259204865,
+      "logits/rejected": -0.13863089680671692,
+      "logps/chosen": -1.454467535018921,
+      "logps/rejected": -2.1822597980499268,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.454467535018921,
+      "rewards/margins": 0.7277923822402954,
+      "rewards/rejected": -2.1822597980499268,
+      "sft_loss": 1.5677722692489624,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 5.935104918288228,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": -0.3364775478839874,
+      "logits/rejected": -0.2209477722644806,
+      "logps/chosen": -1.5508983135223389,
+      "logps/rejected": -2.298210620880127,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5508983135223389,
+      "rewards/margins": 0.7473124265670776,
+      "rewards/rejected": -2.298210620880127,
+      "sft_loss": 1.656023621559143,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 8.690393763781572,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": -0.191738560795784,
+      "logits/rejected": -0.04482860490679741,
+      "logps/chosen": -1.5055210590362549,
+      "logps/rejected": -2.2219274044036865,
+      "loss": 0.4312,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5055210590362549,
+      "rewards/margins": 0.7164064645767212,
+      "rewards/rejected": -2.2219274044036865,
+      "sft_loss": 1.5471540689468384,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 8.615821035292372,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": -0.04971639811992645,
+      "logits/rejected": 0.04083705693483353,
+      "logps/chosen": -1.4413139820098877,
+      "logps/rejected": -2.260993480682373,
+      "loss": 0.4066,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4413139820098877,
+      "rewards/margins": 0.8196796178817749,
+      "rewards/rejected": -2.260993480682373,
+      "sft_loss": 1.5663115978240967,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 9.24558337178255,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": -0.15009254217147827,
+      "logits/rejected": -0.01788342371582985,
+      "logps/chosen": -1.460079550743103,
+      "logps/rejected": -2.1505239009857178,
+      "loss": 0.452,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.460079550743103,
+      "rewards/margins": 0.6904444694519043,
+      "rewards/rejected": -2.1505239009857178,
+      "sft_loss": 1.5730839967727661,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 11.280343455224969,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": -0.13067641854286194,
+      "logits/rejected": -0.04214060679078102,
+      "logps/chosen": -1.6614328622817993,
+      "logps/rejected": -2.3834664821624756,
+      "loss": 0.4509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6614328622817993,
+      "rewards/margins": 0.7220335602760315,
+      "rewards/rejected": -2.3834664821624756,
+      "sft_loss": 1.8111798763275146,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 10.464309113105843,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": -0.30967697501182556,
+      "logits/rejected": -0.1269526481628418,
+      "logps/chosen": -1.6145133972167969,
+      "logps/rejected": -2.3082594871520996,
+      "loss": 0.4437,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6145133972167969,
+      "rewards/margins": 0.6937457919120789,
+      "rewards/rejected": -2.3082594871520996,
+      "sft_loss": 1.753544807434082,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 6.449572000966775,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": -0.23348002135753632,
+      "logits/rejected": -0.13083966076374054,
+      "logps/chosen": -1.6567020416259766,
+      "logps/rejected": -2.081791400909424,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6567020416259766,
+      "rewards/margins": 0.42508941888809204,
+      "rewards/rejected": -2.081791400909424,
+      "sft_loss": 1.7725639343261719,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 7.184053345840208,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": -0.16638872027397156,
+      "logits/rejected": -0.01653314381837845,
+      "logps/chosen": -1.5108344554901123,
+      "logps/rejected": -2.2297990322113037,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5108344554901123,
+      "rewards/margins": 0.7189643383026123,
+      "rewards/rejected": -2.2297990322113037,
+      "sft_loss": 1.647355318069458,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 8.670758590717718,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": -0.22268524765968323,
+      "logits/rejected": -0.03691389039158821,
+      "logps/chosen": -1.6005432605743408,
+      "logps/rejected": -2.360244035720825,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6005432605743408,
+      "rewards/margins": 0.7597008943557739,
+      "rewards/rejected": -2.360244035720825,
+      "sft_loss": 1.785644292831421,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 8.260339247335292,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": -0.2165859490633011,
+      "logits/rejected": -0.23379795253276825,
+      "logps/chosen": -1.51551353931427,
+      "logps/rejected": -2.400438070297241,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.51551353931427,
+      "rewards/margins": 0.884924590587616,
+      "rewards/rejected": -2.400438070297241,
+      "sft_loss": 1.5888447761535645,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 10.01943548791071,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": -0.16946204006671906,
+      "logits/rejected": -0.03227314352989197,
+      "logps/chosen": -1.4113956689834595,
+      "logps/rejected": -2.093937635421753,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4113956689834595,
+      "rewards/margins": 0.6825419664382935,
+      "rewards/rejected": -2.093937635421753,
+      "sft_loss": 1.5719985961914062,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 7.831384054268323,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": -0.178746297955513,
+      "logits/rejected": -0.07838728278875351,
+      "logps/chosen": -1.5404300689697266,
+      "logps/rejected": -2.182508945465088,
+      "loss": 0.461,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5404300689697266,
+      "rewards/margins": 0.642078697681427,
+      "rewards/rejected": -2.182508945465088,
+      "sft_loss": 1.6610815525054932,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 8.281073335530952,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": -0.39260661602020264,
+      "logits/rejected": -0.13857212662696838,
+      "logps/chosen": -1.5768979787826538,
+      "logps/rejected": -2.304960250854492,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5768979787826538,
+      "rewards/margins": 0.7280624508857727,
+      "rewards/rejected": -2.304960250854492,
+      "sft_loss": 1.7252795696258545,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 9.723847297600088,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": -0.1615389585494995,
+      "logits/rejected": 0.019265184178948402,
+      "logps/chosen": -1.606909990310669,
+      "logps/rejected": -2.394131898880005,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.606909990310669,
+      "rewards/margins": 0.7872219681739807,
+      "rewards/rejected": -2.394131898880005,
+      "sft_loss": 1.6091573238372803,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 9.830988256385965,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": -0.25972622632980347,
+      "logits/rejected": -0.08407093584537506,
+      "logps/chosen": -1.5240100622177124,
+      "logps/rejected": -2.0847277641296387,
+      "loss": 0.485,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5240100622177124,
+      "rewards/margins": 0.5607175230979919,
+      "rewards/rejected": -2.0847277641296387,
+      "sft_loss": 1.6416919231414795,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 6.381974017322839,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": -0.20109012722969055,
+      "logits/rejected": -0.09972207248210907,
+      "logps/chosen": -1.6247422695159912,
+      "logps/rejected": -2.0559937953948975,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6247422695159912,
+      "rewards/margins": 0.43125152587890625,
+      "rewards/rejected": -2.0559937953948975,
+      "sft_loss": 1.7206628322601318,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 9.754375916741807,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": -0.24837973713874817,
+      "logits/rejected": -0.027765735983848572,
+      "logps/chosen": -1.5064687728881836,
+      "logps/rejected": -2.308729887008667,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5064687728881836,
+      "rewards/margins": 0.802261471748352,
+      "rewards/rejected": -2.308729887008667,
+      "sft_loss": 1.6191641092300415,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 6.164449381414046,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": -0.24197252094745636,
+      "logits/rejected": -0.0956760048866272,
+      "logps/chosen": -1.5101855993270874,
+      "logps/rejected": -2.1189210414886475,
+      "loss": 0.4513,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5101855993270874,
+      "rewards/margins": 0.6087356805801392,
+      "rewards/rejected": -2.1189210414886475,
+      "sft_loss": 1.5725116729736328,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 8.143842354509493,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": -0.19666540622711182,
+      "logits/rejected": -0.19075897336006165,
+      "logps/chosen": -1.4913088083267212,
+      "logps/rejected": -2.094479560852051,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4913088083267212,
+      "rewards/margins": 0.60317063331604,
+      "rewards/rejected": -2.094479560852051,
+      "sft_loss": 1.59578275680542,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 11.325967769963212,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": -0.1340693086385727,
+      "logits/rejected": -0.018059438094496727,
+      "logps/chosen": -1.5090904235839844,
+      "logps/rejected": -2.183967113494873,
+      "loss": 0.4342,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5090904235839844,
+      "rewards/margins": 0.6748765110969543,
+      "rewards/rejected": -2.183967113494873,
+      "sft_loss": 1.553903341293335,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 7.159332913727375,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": -0.24333210289478302,
+      "logits/rejected": -0.10442526638507843,
+      "logps/chosen": -1.4497852325439453,
+      "logps/rejected": -2.13478422164917,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4497852325439453,
+      "rewards/margins": 0.6849991083145142,
+      "rewards/rejected": -2.13478422164917,
+      "sft_loss": 1.6094410419464111,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 11.093852678407805,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": -0.15132607519626617,
+      "logits/rejected": -0.07192808389663696,
+      "logps/chosen": -1.4410126209259033,
+      "logps/rejected": -2.1370604038238525,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4410126209259033,
+      "rewards/margins": 0.6960476636886597,
+      "rewards/rejected": -2.1370604038238525,
+      "sft_loss": 1.4817826747894287,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 8.217590422474169,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": -0.1982906460762024,
+      "logits/rejected": -0.13623209297657013,
+      "logps/chosen": -1.5887467861175537,
+      "logps/rejected": -2.1640536785125732,
+      "loss": 0.4649,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5887467861175537,
+      "rewards/margins": 0.5753068923950195,
+      "rewards/rejected": -2.1640536785125732,
+      "sft_loss": 1.6395225524902344,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 7.250556524321402,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": -0.14488236606121063,
+      "logits/rejected": -0.067586749792099,
+      "logps/chosen": -1.7181322574615479,
+      "logps/rejected": -2.3087222576141357,
+      "loss": 0.517,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7181322574615479,
+      "rewards/margins": 0.5905901193618774,
+      "rewards/rejected": -2.3087222576141357,
+      "sft_loss": 1.6792157888412476,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 9.52426655213416,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": -0.15554898977279663,
+      "logits/rejected": 0.04365387558937073,
+      "logps/chosen": -1.4417258501052856,
+      "logps/rejected": -2.2459707260131836,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4417258501052856,
+      "rewards/margins": 0.8042449951171875,
+      "rewards/rejected": -2.2459707260131836,
+      "sft_loss": 1.4861071109771729,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 11.527604544943122,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": -0.3058409094810486,
+      "logits/rejected": -0.07154044508934021,
+      "logps/chosen": -1.518927812576294,
+      "logps/rejected": -2.209139108657837,
+      "loss": 0.4262,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.518927812576294,
+      "rewards/margins": 0.690211296081543,
+      "rewards/rejected": -2.209139108657837,
+      "sft_loss": 1.616014838218689,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 7.802385369560936,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": -0.24710774421691895,
+      "logits/rejected": -0.2252301722764969,
+      "logps/chosen": -1.5771944522857666,
+      "logps/rejected": -2.0971343517303467,
+      "loss": 0.466,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5771944522857666,
+      "rewards/margins": 0.5199397802352905,
+      "rewards/rejected": -2.0971343517303467,
+      "sft_loss": 1.6544469594955444,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 6.263767299351617,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": -0.22472020983695984,
+      "logits/rejected": -0.20360127091407776,
+      "logps/chosen": -1.6000200510025024,
+      "logps/rejected": -2.1722025871276855,
+      "loss": 0.4853,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6000200510025024,
+      "rewards/margins": 0.5721826553344727,
+      "rewards/rejected": -2.1722025871276855,
+      "sft_loss": 1.6936304569244385,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 7.991459840747118,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": -0.17534606158733368,
+      "logits/rejected": -0.12202417850494385,
+      "logps/chosen": -1.537259817123413,
+      "logps/rejected": -2.1599433422088623,
+      "loss": 0.4254,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.537259817123413,
+      "rewards/margins": 0.6226836442947388,
+      "rewards/rejected": -2.1599433422088623,
+      "sft_loss": 1.598702073097229,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 8.166723627192075,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": -0.14732176065444946,
+      "logits/rejected": -0.03624642640352249,
+      "logps/chosen": -1.6838470697402954,
+      "logps/rejected": -2.3584134578704834,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6838470697402954,
+      "rewards/margins": 0.6745663285255432,
+      "rewards/rejected": -2.3584134578704834,
+      "sft_loss": 1.7807865142822266,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 10.626354095511608,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": -0.25213223695755005,
+      "logits/rejected": -0.1784714162349701,
+      "logps/chosen": -1.581032395362854,
+      "logps/rejected": -2.1678004264831543,
+      "loss": 0.4511,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.581032395362854,
+      "rewards/margins": 0.5867680311203003,
+      "rewards/rejected": -2.1678004264831543,
+      "sft_loss": 1.6110156774520874,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 9.717407654604258,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": -0.26242008805274963,
+      "logits/rejected": -0.044322580099105835,
+      "logps/chosen": -1.4938757419586182,
+      "logps/rejected": -2.065871000289917,
+      "loss": 0.4531,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4938757419586182,
+      "rewards/margins": 0.5719951391220093,
+      "rewards/rejected": -2.065871000289917,
+      "sft_loss": 1.5849002599716187,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 8.53375616791577,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": -0.25997504591941833,
+      "logits/rejected": -0.10667125880718231,
+      "logps/chosen": -1.5702990293502808,
+      "logps/rejected": -2.188900947570801,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5702990293502808,
+      "rewards/margins": 0.6186016798019409,
+      "rewards/rejected": -2.188900947570801,
+      "sft_loss": 1.6220414638519287,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 8.298187880845026,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": -0.19612005352973938,
+      "logits/rejected": -0.11876839399337769,
+      "logps/chosen": -1.5246912240982056,
+      "logps/rejected": -2.186675548553467,
+      "loss": 0.474,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5246912240982056,
+      "rewards/margins": 0.6619843244552612,
+      "rewards/rejected": -2.186675548553467,
+      "sft_loss": 1.5625849962234497,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 17.42258631649615,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": -0.2558310627937317,
+      "logits/rejected": -0.12091462314128876,
+      "logps/chosen": -1.6476072072982788,
+      "logps/rejected": -2.227116346359253,
+      "loss": 0.4678,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6476072072982788,
+      "rewards/margins": 0.5795091390609741,
+      "rewards/rejected": -2.227116346359253,
+      "sft_loss": 1.7288262844085693,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 9.098056259420295,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": -0.035765133798122406,
+      "logits/rejected": -0.07453829795122147,
+      "logps/chosen": -1.5224034786224365,
+      "logps/rejected": -2.1308224201202393,
+      "loss": 0.4588,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5224034786224365,
+      "rewards/margins": 0.6084188222885132,
+      "rewards/rejected": -2.1308224201202393,
+      "sft_loss": 1.5826609134674072,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 14.179261364612977,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": -0.1643834412097931,
+      "logits/rejected": -0.1282930076122284,
+      "logps/chosen": -1.4879213571548462,
+      "logps/rejected": -2.031446933746338,
+      "loss": 0.4554,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4879213571548462,
+      "rewards/margins": 0.5435255169868469,
+      "rewards/rejected": -2.031446933746338,
+      "sft_loss": 1.5665534734725952,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 7.54749645668722,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": -0.23299995064735413,
+      "logits/rejected": -0.15355482697486877,
+      "logps/chosen": -1.3718222379684448,
+      "logps/rejected": -1.998819351196289,
+      "loss": 0.4129,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.3718222379684448,
+      "rewards/margins": 0.6269969344139099,
+      "rewards/rejected": -1.998819351196289,
+      "sft_loss": 1.4779876470565796,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 9.293253168136241,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": -0.3311172425746918,
+      "logits/rejected": -0.24270012974739075,
+      "logps/chosen": -1.3977210521697998,
+      "logps/rejected": -2.0432395935058594,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3977210521697998,
+      "rewards/margins": 0.6455187797546387,
+      "rewards/rejected": -2.0432395935058594,
+      "sft_loss": 1.497981071472168,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 11.595698661147347,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": -0.16359257698059082,
+      "logits/rejected": -0.09543822705745697,
+      "logps/chosen": -1.5558207035064697,
+      "logps/rejected": -2.118010997772217,
+      "loss": 0.4694,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5558207035064697,
+      "rewards/margins": 0.5621901750564575,
+      "rewards/rejected": -2.118010997772217,
+      "sft_loss": 1.6672084331512451,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 14.84432234803869,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": -0.26126575469970703,
+      "logits/rejected": -0.00785075407475233,
+      "logps/chosen": -1.5473171472549438,
+      "logps/rejected": -2.1697630882263184,
+      "loss": 0.4402,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5473171472549438,
+      "rewards/margins": 0.6224457025527954,
+      "rewards/rejected": -2.1697630882263184,
+      "sft_loss": 1.6175849437713623,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 7.8924081270315725,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": -0.10214686393737793,
+      "logits/rejected": 0.0706239640712738,
+      "logps/chosen": -1.4367588758468628,
+      "logps/rejected": -1.9674361944198608,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4367588758468628,
+      "rewards/margins": 0.5306774377822876,
+      "rewards/rejected": -1.9674361944198608,
+      "sft_loss": 1.5512819290161133,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 13.96875352505859,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": -0.21306836605072021,
+      "logits/rejected": -0.12876783311367035,
+      "logps/chosen": -1.5248085260391235,
+      "logps/rejected": -1.9554111957550049,
+      "loss": 0.502,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5248085260391235,
+      "rewards/margins": 0.4306026101112366,
+      "rewards/rejected": -1.9554111957550049,
+      "sft_loss": 1.6193593740463257,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 7.563130631641448,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": -0.2365640103816986,
+      "logits/rejected": -0.08295810222625732,
+      "logps/chosen": -1.5342161655426025,
+      "logps/rejected": -2.123009204864502,
+      "loss": 0.4521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5342161655426025,
+      "rewards/margins": 0.5887928605079651,
+      "rewards/rejected": -2.123009204864502,
+      "sft_loss": 1.6141750812530518,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 14.560564045470185,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": -0.2672147750854492,
+      "logits/rejected": -0.13339033722877502,
+      "logps/chosen": -1.4195791482925415,
+      "logps/rejected": -2.135160446166992,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4195791482925415,
+      "rewards/margins": 0.7155812978744507,
+      "rewards/rejected": -2.135160446166992,
+      "sft_loss": 1.4889144897460938,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 11.143544002117181,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": -0.34407171607017517,
+      "logits/rejected": -0.16657240688800812,
+      "logps/chosen": -1.5777204036712646,
+      "logps/rejected": -2.2593626976013184,
+      "loss": 0.4461,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5777204036712646,
+      "rewards/margins": 0.6816423535346985,
+      "rewards/rejected": -2.2593626976013184,
+      "sft_loss": 1.6737632751464844,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 9.455706044788442,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": -0.35349464416503906,
+      "logits/rejected": -0.1343749463558197,
+      "logps/chosen": -1.5024068355560303,
+      "logps/rejected": -2.2888574600219727,
+      "loss": 0.4242,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5024068355560303,
+      "rewards/margins": 0.7864503860473633,
+      "rewards/rejected": -2.2888574600219727,
+      "sft_loss": 1.5991122722625732,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 7.712027144423693,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": -0.13933254778385162,
+      "logits/rejected": 0.02814735472202301,
+      "logps/chosen": -1.4643703699111938,
+      "logps/rejected": -2.3610987663269043,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4643703699111938,
+      "rewards/margins": 0.8967283964157104,
+      "rewards/rejected": -2.3610987663269043,
+      "sft_loss": 1.5425755977630615,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 11.281307044328882,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": -0.20330731570720673,
+      "logits/rejected": -0.023081596940755844,
+      "logps/chosen": -1.5841712951660156,
+      "logps/rejected": -2.14113712310791,
+      "loss": 0.4599,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5841712951660156,
+      "rewards/margins": 0.5569659471511841,
+      "rewards/rejected": -2.14113712310791,
+      "sft_loss": 1.6328054666519165,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 9.18953732538059,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": -0.28744104504585266,
+      "logits/rejected": -0.13670162856578827,
+      "logps/chosen": -1.5169239044189453,
+      "logps/rejected": -2.094639539718628,
+      "loss": 0.4631,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5169239044189453,
+      "rewards/margins": 0.5777156352996826,
+      "rewards/rejected": -2.094639539718628,
+      "sft_loss": 1.5577342510223389,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 6.65679864485864,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": -0.30195218324661255,
+      "logits/rejected": -0.06154744699597359,
+      "logps/chosen": -1.5235546827316284,
+      "logps/rejected": -2.314807415008545,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5235546827316284,
+      "rewards/margins": 0.7912526726722717,
+      "rewards/rejected": -2.314807415008545,
+      "sft_loss": 1.599188208580017,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 9.923834819757321,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": -0.24068517982959747,
+      "logits/rejected": -0.1709970235824585,
+      "logps/chosen": -1.641798734664917,
+      "logps/rejected": -2.3314099311828613,
+      "loss": 0.4287,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.641798734664917,
+      "rewards/margins": 0.6896110773086548,
+      "rewards/rejected": -2.3314099311828613,
+      "sft_loss": 1.6708080768585205,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 23.02297917390171,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": -0.35668811202049255,
+      "logits/rejected": -0.1095414012670517,
+      "logps/chosen": -1.5242853164672852,
+      "logps/rejected": -2.1377031803131104,
+      "loss": 0.445,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5242853164672852,
+      "rewards/margins": 0.6134177446365356,
+      "rewards/rejected": -2.1377031803131104,
+      "sft_loss": 1.6786664724349976,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 6.036728353405439,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": -0.3969551920890808,
+      "logits/rejected": -0.189706951379776,
+      "logps/chosen": -1.5305182933807373,
+      "logps/rejected": -2.223459482192993,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5305182933807373,
+      "rewards/margins": 0.6929410099983215,
+      "rewards/rejected": -2.223459482192993,
+      "sft_loss": 1.66098952293396,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 10.095901836204533,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": -0.16776272654533386,
+      "logits/rejected": -0.08403909206390381,
+      "logps/chosen": -1.5309851169586182,
+      "logps/rejected": -2.127441883087158,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5309851169586182,
+      "rewards/margins": 0.5964566469192505,
+      "rewards/rejected": -2.127441883087158,
+      "sft_loss": 1.6376146078109741,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 8.905118245131481,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": -0.18821024894714355,
+      "logits/rejected": -0.03925692290067673,
+      "logps/chosen": -1.6175647974014282,
+      "logps/rejected": -2.1396234035491943,
+      "loss": 0.4854,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6175647974014282,
+      "rewards/margins": 0.52205890417099,
+      "rewards/rejected": -2.1396234035491943,
+      "sft_loss": 1.666550636291504,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 8.405618016929523,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": -0.258244127035141,
+      "logits/rejected": -0.07392361015081406,
+      "logps/chosen": -1.4821580648422241,
+      "logps/rejected": -2.103391170501709,
+      "loss": 0.4643,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4821580648422241,
+      "rewards/margins": 0.6212331056594849,
+      "rewards/rejected": -2.103391170501709,
+      "sft_loss": 1.6087650060653687,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 5.673376367465625,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": -0.3048526346683502,
+      "logits/rejected": -0.1681317389011383,
+      "logps/chosen": -1.4941984415054321,
+      "logps/rejected": -2.1286990642547607,
+      "loss": 0.42,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4941984415054321,
+      "rewards/margins": 0.6345007419586182,
+      "rewards/rejected": -2.1286990642547607,
+      "sft_loss": 1.573027491569519,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 8.00899310290481,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": -0.3116549849510193,
+      "logits/rejected": -0.19156138598918915,
+      "logps/chosen": -1.551615834236145,
+      "logps/rejected": -2.102240800857544,
+      "loss": 0.4607,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.551615834236145,
+      "rewards/margins": 0.5506250858306885,
+      "rewards/rejected": -2.102240800857544,
+      "sft_loss": 1.6373153924942017,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 5.466003891775965,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": -0.2839297652244568,
+      "logits/rejected": -0.21033921837806702,
+      "logps/chosen": -1.5315322875976562,
+      "logps/rejected": -2.277865409851074,
+      "loss": 0.4046,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5315322875976562,
+      "rewards/margins": 0.7463330030441284,
+      "rewards/rejected": -2.277865409851074,
+      "sft_loss": 1.680361032485962,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 10.590259452652054,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": -0.23004582524299622,
+      "logits/rejected": -0.027688121423125267,
+      "logps/chosen": -1.4334943294525146,
+      "logps/rejected": -2.082756757736206,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4334943294525146,
+      "rewards/margins": 0.6492623686790466,
+      "rewards/rejected": -2.082756757736206,
+      "sft_loss": 1.511309027671814,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 6.041999725545914,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": -0.2488495111465454,
+      "logits/rejected": -0.10540600121021271,
+      "logps/chosen": -1.5621020793914795,
+      "logps/rejected": -2.1619415283203125,
+      "loss": 0.4774,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5621020793914795,
+      "rewards/margins": 0.5998395085334778,
+      "rewards/rejected": -2.1619415283203125,
+      "sft_loss": 1.6926653385162354,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 7.86482733723809,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": -0.266471266746521,
+      "logits/rejected": -0.14478352665901184,
+      "logps/chosen": -1.5819045305252075,
+      "logps/rejected": -2.25170636177063,
+      "loss": 0.4435,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5819045305252075,
+      "rewards/margins": 0.6698015332221985,
+      "rewards/rejected": -2.25170636177063,
+      "sft_loss": 1.6461341381072998,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 8.961045368620963,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": -0.17507778108119965,
+      "logits/rejected": -0.10742446035146713,
+      "logps/chosen": -1.6066890954971313,
+      "logps/rejected": -2.258471965789795,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6066890954971313,
+      "rewards/margins": 0.6517831087112427,
+      "rewards/rejected": -2.258471965789795,
+      "sft_loss": 1.7125810384750366,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 7.230794407833966,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": -0.1980207860469818,
+      "logits/rejected": -0.0723222941160202,
+      "logps/chosen": -1.4438279867172241,
+      "logps/rejected": -2.1929969787597656,
+      "loss": 0.4185,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4438279867172241,
+      "rewards/margins": 0.7491689324378967,
+      "rewards/rejected": -2.1929969787597656,
+      "sft_loss": 1.5771210193634033,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 8.572718412601594,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": -0.3288986384868622,
+      "logits/rejected": -0.14068400859832764,
+      "logps/chosen": -1.4717166423797607,
+      "logps/rejected": -2.157830238342285,
+      "loss": 0.4282,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4717166423797607,
+      "rewards/margins": 0.686113715171814,
+      "rewards/rejected": -2.157830238342285,
+      "sft_loss": 1.6161715984344482,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 10.395910525815902,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": -0.2193346917629242,
+      "logits/rejected": -0.07468325644731522,
+      "logps/chosen": -1.5009483098983765,
+      "logps/rejected": -2.134226083755493,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5009483098983765,
+      "rewards/margins": 0.633277952671051,
+      "rewards/rejected": -2.134226083755493,
+      "sft_loss": 1.554665446281433,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 11.82919190666528,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": -0.31428205966949463,
+      "logits/rejected": -0.13497485220432281,
+      "logps/chosen": -1.517218828201294,
+      "logps/rejected": -2.152031898498535,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.517218828201294,
+      "rewards/margins": 0.634813129901886,
+      "rewards/rejected": -2.152031898498535,
+      "sft_loss": 1.6881401538848877,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": 0.13055913150310516,
+      "eval_logits/rejected": 0.23271715641021729,
+      "eval_logps/chosen": -1.6105517148971558,
+      "eval_logps/rejected": -2.1856274604797363,
+      "eval_loss": 0.48953157663345337,
+      "eval_rewards/accuracies": 0.6706231236457825,
+      "eval_rewards/chosen": -1.6105517148971558,
+      "eval_rewards/margins": 0.5750758647918701,
+      "eval_rewards/rejected": -2.1856274604797363,
+      "eval_runtime": 43.0982,
+      "eval_samples_per_second": 31.208,
+      "eval_sft_loss": 1.6850907802581787,
+      "eval_steps_per_second": 7.819,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 5.776006694489402,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": -0.16184869408607483,
+      "logits/rejected": -0.11565609276294708,
+      "logps/chosen": -1.522731065750122,
+      "logps/rejected": -2.048495292663574,
+      "loss": 0.4919,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.522731065750122,
+      "rewards/margins": 0.525763988494873,
+      "rewards/rejected": -2.048495292663574,
+      "sft_loss": 1.5864933729171753,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 11.013142341793925,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": -0.23087449371814728,
+      "logits/rejected": -0.12524442374706268,
+      "logps/chosen": -1.5012990236282349,
+      "logps/rejected": -2.2111098766326904,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5012990236282349,
+      "rewards/margins": 0.7098108530044556,
+      "rewards/rejected": -2.2111098766326904,
+      "sft_loss": 1.5755040645599365,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 6.685725664190028,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": -0.2300528585910797,
+      "logits/rejected": -0.11930929124355316,
+      "logps/chosen": -1.4321932792663574,
+      "logps/rejected": -2.1179568767547607,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4321932792663574,
+      "rewards/margins": 0.6857637166976929,
+      "rewards/rejected": -2.1179568767547607,
+      "sft_loss": 1.5366168022155762,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 10.542810332554014,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": -0.26986294984817505,
+      "logits/rejected": -0.18295058608055115,
+      "logps/chosen": -1.4609721899032593,
+      "logps/rejected": -2.1687216758728027,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4609721899032593,
+      "rewards/margins": 0.7077494263648987,
+      "rewards/rejected": -2.1687216758728027,
+      "sft_loss": 1.616141676902771,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 11.314222661472881,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": -0.17271623015403748,
+      "logits/rejected": -0.020607611164450645,
+      "logps/chosen": -1.5896047353744507,
+      "logps/rejected": -2.263044834136963,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5896047353744507,
+      "rewards/margins": 0.6734402179718018,
+      "rewards/rejected": -2.263044834136963,
+      "sft_loss": 1.6785653829574585,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 8.33161220717835,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": -0.3504981994628906,
+      "logits/rejected": -0.19691799581050873,
+      "logps/chosen": -1.5263431072235107,
+      "logps/rejected": -2.1571273803710938,
+      "loss": 0.4404,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5263431072235107,
+      "rewards/margins": 0.6307845711708069,
+      "rewards/rejected": -2.1571273803710938,
+      "sft_loss": 1.58456289768219,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 8.253808280426956,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": -0.294038861989975,
+      "logits/rejected": -0.15794646739959717,
+      "logps/chosen": -1.5321094989776611,
+      "logps/rejected": -2.1294748783111572,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5321094989776611,
+      "rewards/margins": 0.5973653793334961,
+      "rewards/rejected": -2.1294748783111572,
+      "sft_loss": 1.5454280376434326,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 8.614401323815699,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": -0.28644105792045593,
+      "logits/rejected": -0.0708000659942627,
+      "logps/chosen": -1.5604805946350098,
+      "logps/rejected": -2.216197967529297,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5604805946350098,
+      "rewards/margins": 0.6557173132896423,
+      "rewards/rejected": -2.216197967529297,
+      "sft_loss": 1.628729224205017,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 8.606419640888854,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": -0.16988658905029297,
+      "logits/rejected": -0.11387205123901367,
+      "logps/chosen": -1.4814785718917847,
+      "logps/rejected": -2.048768997192383,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4814785718917847,
+      "rewards/margins": 0.5672904253005981,
+      "rewards/rejected": -2.048768997192383,
+      "sft_loss": 1.5562591552734375,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 12.649362573939653,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": -0.3165106773376465,
+      "logits/rejected": -0.15842103958129883,
+      "logps/chosen": -1.6255061626434326,
+      "logps/rejected": -2.3039708137512207,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6255061626434326,
+      "rewards/margins": 0.6784645915031433,
+      "rewards/rejected": -2.3039708137512207,
+      "sft_loss": 1.7424743175506592,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 12.29929108395042,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": -0.2662120759487152,
+      "logits/rejected": -0.02815375290811062,
+      "logps/chosen": -1.5816794633865356,
+      "logps/rejected": -2.2056572437286377,
+      "loss": 0.4591,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5816794633865356,
+      "rewards/margins": 0.6239776015281677,
+      "rewards/rejected": -2.2056572437286377,
+      "sft_loss": 1.6130762100219727,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 8.28428810284338,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": -0.22748716175556183,
+      "logits/rejected": -0.1518431007862091,
+      "logps/chosen": -1.5503032207489014,
+      "logps/rejected": -2.2076878547668457,
+      "loss": 0.4621,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5503032207489014,
+      "rewards/margins": 0.6573843359947205,
+      "rewards/rejected": -2.2076878547668457,
+      "sft_loss": 1.7244062423706055,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 8.376139146060785,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": -0.38391146063804626,
+      "logits/rejected": -0.13936465978622437,
+      "logps/chosen": -1.784136414527893,
+      "logps/rejected": -2.388622760772705,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.784136414527893,
+      "rewards/margins": 0.604486346244812,
+      "rewards/rejected": -2.388622760772705,
+      "sft_loss": 1.860600471496582,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 7.011574259425963,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": -0.23148080706596375,
+      "logits/rejected": -0.07034826278686523,
+      "logps/chosen": -1.5404936075210571,
+      "logps/rejected": -2.229719638824463,
+      "loss": 0.4208,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.5404936075210571,
+      "rewards/margins": 0.6892261505126953,
+      "rewards/rejected": -2.229719638824463,
+      "sft_loss": 1.607832670211792,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 7.240883226597204,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": -0.2358875274658203,
+      "logits/rejected": -0.07125989347696304,
+      "logps/chosen": -1.6322225332260132,
+      "logps/rejected": -2.355457067489624,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6322225332260132,
+      "rewards/margins": 0.7232344746589661,
+      "rewards/rejected": -2.355457067489624,
+      "sft_loss": 1.6577503681182861,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 9.271934661143398,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": -0.24580411612987518,
+      "logits/rejected": -0.0542788989841938,
+      "logps/chosen": -1.715682029724121,
+      "logps/rejected": -2.4755477905273438,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.715682029724121,
+      "rewards/margins": 0.7598655819892883,
+      "rewards/rejected": -2.4755477905273438,
+      "sft_loss": 1.7498623132705688,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 11.374744424713278,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": -0.30271807312965393,
+      "logits/rejected": -0.1887938678264618,
+      "logps/chosen": -1.4635353088378906,
+      "logps/rejected": -2.2433948516845703,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4635353088378906,
+      "rewards/margins": 0.7798596620559692,
+      "rewards/rejected": -2.2433948516845703,
+      "sft_loss": 1.5233333110809326,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 6.872953005493221,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": -0.2544511556625366,
+      "logits/rejected": -0.096409872174263,
+      "logps/chosen": -1.5046459436416626,
+      "logps/rejected": -2.1032772064208984,
+      "loss": 0.4494,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5046459436416626,
+      "rewards/margins": 0.5986312031745911,
+      "rewards/rejected": -2.1032772064208984,
+      "sft_loss": 1.6099576950073242,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 6.137494826088779,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": -0.3068126440048218,
+      "logits/rejected": -0.09982781112194061,
+      "logps/chosen": -1.6437880992889404,
+      "logps/rejected": -2.3714306354522705,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6437880992889404,
+      "rewards/margins": 0.7276423573493958,
+      "rewards/rejected": -2.3714306354522705,
+      "sft_loss": 1.72063410282135,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 9.158112466999524,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": -0.26331356167793274,
+      "logits/rejected": -0.21247005462646484,
+      "logps/chosen": -1.5546457767486572,
+      "logps/rejected": -2.288158893585205,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5546457767486572,
+      "rewards/margins": 0.7335132360458374,
+      "rewards/rejected": -2.288158893585205,
+      "sft_loss": 1.5880502462387085,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 8.260041502015017,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": -0.2249482423067093,
+      "logits/rejected": -0.07068298757076263,
+      "logps/chosen": -1.59334397315979,
+      "logps/rejected": -2.2703146934509277,
+      "loss": 0.4335,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.59334397315979,
+      "rewards/margins": 0.6769706606864929,
+      "rewards/rejected": -2.2703146934509277,
+      "sft_loss": 1.7753381729125977,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 8.102204753805093,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": -0.3869312107563019,
+      "logits/rejected": -0.12273693084716797,
+      "logps/chosen": -1.4944441318511963,
+      "logps/rejected": -2.322719097137451,
+      "loss": 0.4284,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4944441318511963,
+      "rewards/margins": 0.8282749056816101,
+      "rewards/rejected": -2.322719097137451,
+      "sft_loss": 1.5989134311676025,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 10.578547394286801,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": -0.13628318905830383,
+      "logits/rejected": 0.0010598197113722563,
+      "logps/chosen": -1.5226942300796509,
+      "logps/rejected": -2.0845484733581543,
+      "loss": 0.4386,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5226942300796509,
+      "rewards/margins": 0.5618543028831482,
+      "rewards/rejected": -2.0845484733581543,
+      "sft_loss": 1.6184759140014648,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 6.059569829129419,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": -0.15193864703178406,
+      "logits/rejected": -0.059926360845565796,
+      "logps/chosen": -1.4939788579940796,
+      "logps/rejected": -2.059293270111084,
+      "loss": 0.4537,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4939788579940796,
+      "rewards/margins": 0.565314531326294,
+      "rewards/rejected": -2.059293270111084,
+      "sft_loss": 1.5195624828338623,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 6.775886783493703,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": -0.21538333594799042,
+      "logits/rejected": -0.05382348224520683,
+      "logps/chosen": -1.5950740575790405,
+      "logps/rejected": -2.3566675186157227,
+      "loss": 0.4172,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5950740575790405,
+      "rewards/margins": 0.7615932822227478,
+      "rewards/rejected": -2.3566675186157227,
+      "sft_loss": 1.656071424484253,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 10.365899347234771,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": -0.15810880064964294,
+      "logits/rejected": -0.06887652724981308,
+      "logps/chosen": -1.4541465044021606,
+      "logps/rejected": -2.1913342475891113,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4541465044021606,
+      "rewards/margins": 0.7371879816055298,
+      "rewards/rejected": -2.1913342475891113,
+      "sft_loss": 1.5949485301971436,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 11.959122542324632,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": -0.10423209518194199,
+      "logits/rejected": -0.12887991964817047,
+      "logps/chosen": -1.5037696361541748,
+      "logps/rejected": -1.9653781652450562,
+      "loss": 0.5121,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5037696361541748,
+      "rewards/margins": 0.46160855889320374,
+      "rewards/rejected": -1.9653781652450562,
+      "sft_loss": 1.541277527809143,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 7.756715618841046,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": -0.1794317662715912,
+      "logits/rejected": -0.12930874526500702,
+      "logps/chosen": -1.5389997959136963,
+      "logps/rejected": -2.155897855758667,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5389997959136963,
+      "rewards/margins": 0.6168978810310364,
+      "rewards/rejected": -2.155897855758667,
+      "sft_loss": 1.5519967079162598,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 5.211057613264154,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": -0.3666417896747589,
+      "logits/rejected": -0.17199210822582245,
+      "logps/chosen": -1.5276671648025513,
+      "logps/rejected": -2.213324546813965,
+      "loss": 0.401,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.5276671648025513,
+      "rewards/margins": 0.6856572031974792,
+      "rewards/rejected": -2.213324546813965,
+      "sft_loss": 1.60750412940979,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 8.446061188353697,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": -0.15566708147525787,
+      "logits/rejected": -0.1720820665359497,
+      "logps/chosen": -1.6055930852890015,
+      "logps/rejected": -2.133291721343994,
+      "loss": 0.4838,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6055930852890015,
+      "rewards/margins": 0.5276986360549927,
+      "rewards/rejected": -2.133291721343994,
+      "sft_loss": 1.6225850582122803,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 10.755228467455938,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": -0.27801212668418884,
+      "logits/rejected": -0.18713174760341644,
+      "logps/chosen": -1.5514525175094604,
+      "logps/rejected": -2.1258859634399414,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5514525175094604,
+      "rewards/margins": 0.5744332671165466,
+      "rewards/rejected": -2.1258859634399414,
+      "sft_loss": 1.624772071838379,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 12.113195896228424,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": -0.2438342124223709,
+      "logits/rejected": -0.038455717265605927,
+      "logps/chosen": -1.5836608409881592,
+      "logps/rejected": -2.2291040420532227,
+      "loss": 0.4598,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5836608409881592,
+      "rewards/margins": 0.645443320274353,
+      "rewards/rejected": -2.2291040420532227,
+      "sft_loss": 1.6981513500213623,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 8.03902053145534,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": -0.3179672062397003,
+      "logits/rejected": -0.13477668166160583,
+      "logps/chosen": -1.579353928565979,
+      "logps/rejected": -2.28670597076416,
+      "loss": 0.435,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.579353928565979,
+      "rewards/margins": 0.7073520421981812,
+      "rewards/rejected": -2.28670597076416,
+      "sft_loss": 1.6819642782211304,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 8.56323599419119,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": -0.1567922681570053,
+      "logits/rejected": -0.10974302142858505,
+      "logps/chosen": -1.5311553478240967,
+      "logps/rejected": -1.984129548072815,
+      "loss": 0.4859,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5311553478240967,
+      "rewards/margins": 0.45297402143478394,
+      "rewards/rejected": -1.984129548072815,
+      "sft_loss": 1.6002174615859985,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 7.536547968436513,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": -0.22333934903144836,
+      "logits/rejected": -0.11946012079715729,
+      "logps/chosen": -1.4816691875457764,
+      "logps/rejected": -2.308711051940918,
+      "loss": 0.4509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4816691875457764,
+      "rewards/margins": 0.8270419239997864,
+      "rewards/rejected": -2.308711051940918,
+      "sft_loss": 1.6006828546524048,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 7.189079618381418,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": -0.2728548049926758,
+      "logits/rejected": -0.02265390381217003,
+      "logps/chosen": -1.6843229532241821,
+      "logps/rejected": -2.4106035232543945,
+      "loss": 0.427,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6843229532241821,
+      "rewards/margins": 0.7262802124023438,
+      "rewards/rejected": -2.4106035232543945,
+      "sft_loss": 1.7908289432525635,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 5.956115684859829,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": -0.16116848587989807,
+      "logits/rejected": -0.058946527540683746,
+      "logps/chosen": -1.526186227798462,
+      "logps/rejected": -2.0922622680664062,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.526186227798462,
+      "rewards/margins": 0.5660760402679443,
+      "rewards/rejected": -2.0922622680664062,
+      "sft_loss": 1.606529951095581,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 5.904020734651611,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": -0.2092897593975067,
+      "logits/rejected": -0.06519066542387009,
+      "logps/chosen": -1.5699594020843506,
+      "logps/rejected": -2.089143753051758,
+      "loss": 0.4774,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5699594020843506,
+      "rewards/margins": 0.5191842913627625,
+      "rewards/rejected": -2.089143753051758,
+      "sft_loss": 1.6545159816741943,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 7.017257247779873,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": -0.3744376301765442,
+      "logits/rejected": -0.1616448163986206,
+      "logps/chosen": -1.518035650253296,
+      "logps/rejected": -2.1544997692108154,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.518035650253296,
+      "rewards/margins": 0.6364642977714539,
+      "rewards/rejected": -2.1544997692108154,
+      "sft_loss": 1.6293920278549194,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 8.617520189475039,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": -0.283908486366272,
+      "logits/rejected": -0.018505841493606567,
+      "logps/chosen": -1.5354034900665283,
+      "logps/rejected": -2.4752886295318604,
+      "loss": 0.399,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.5354034900665283,
+      "rewards/margins": 0.9398850202560425,
+      "rewards/rejected": -2.4752886295318604,
+      "sft_loss": 1.5902059078216553,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 8.529954126666453,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": -0.18799303472042084,
+      "logits/rejected": -0.13106395304203033,
+      "logps/chosen": -1.7391777038574219,
+      "logps/rejected": -2.3081436157226562,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7391777038574219,
+      "rewards/margins": 0.5689657926559448,
+      "rewards/rejected": -2.3081436157226562,
+      "sft_loss": 1.776010274887085,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 6.961561366310143,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": -0.19545699656009674,
+      "logits/rejected": -0.05803150683641434,
+      "logps/chosen": -1.5638599395751953,
+      "logps/rejected": -2.421759605407715,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5638599395751953,
+      "rewards/margins": 0.8578997850418091,
+      "rewards/rejected": -2.421759605407715,
+      "sft_loss": 1.6521934270858765,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 6.546250379775348,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": -0.28004032373428345,
+      "logits/rejected": -0.08701594918966293,
+      "logps/chosen": -1.5976712703704834,
+      "logps/rejected": -2.1546006202697754,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5976712703704834,
+      "rewards/margins": 0.5569295883178711,
+      "rewards/rejected": -2.1546006202697754,
+      "sft_loss": 1.7328250408172607,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 7.927715183147347,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": -0.19358518719673157,
+      "logits/rejected": -0.11841396987438202,
+      "logps/chosen": -1.5791889429092407,
+      "logps/rejected": -2.156836986541748,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5791889429092407,
+      "rewards/margins": 0.5776482224464417,
+      "rewards/rejected": -2.156836986541748,
+      "sft_loss": 1.697723388671875,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 9.515705477914373,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": -0.22499966621398926,
+      "logits/rejected": -0.07803558558225632,
+      "logps/chosen": -1.5573455095291138,
+      "logps/rejected": -2.2448649406433105,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5573455095291138,
+      "rewards/margins": 0.6875195503234863,
+      "rewards/rejected": -2.2448649406433105,
+      "sft_loss": 1.6308892965316772,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 7.606091817512331,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": -0.2587551772594452,
+      "logits/rejected": -0.07017064839601517,
+      "logps/chosen": -1.4989105463027954,
+      "logps/rejected": -2.214676856994629,
+      "loss": 0.4116,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4989105463027954,
+      "rewards/margins": 0.7157663106918335,
+      "rewards/rejected": -2.214676856994629,
+      "sft_loss": 1.556244134902954,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 11.543911209924728,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": -0.3492546081542969,
+      "logits/rejected": -0.2342105656862259,
+      "logps/chosen": -1.4990334510803223,
+      "logps/rejected": -2.2174575328826904,
+      "loss": 0.4287,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4990334510803223,
+      "rewards/margins": 0.7184239029884338,
+      "rewards/rejected": -2.2174575328826904,
+      "sft_loss": 1.606949806213379,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 8.20323293946449,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": -0.2740515172481537,
+      "logits/rejected": -0.13928236067295074,
+      "logps/chosen": -1.4924064874649048,
+      "logps/rejected": -2.071157217025757,
+      "loss": 0.446,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4924064874649048,
+      "rewards/margins": 0.5787509679794312,
+      "rewards/rejected": -2.071157217025757,
+      "sft_loss": 1.600365400314331,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 7.721124999933021,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": -0.03778085485100746,
+      "logits/rejected": -0.022445034235715866,
+      "logps/chosen": -1.5717326402664185,
+      "logps/rejected": -2.443424701690674,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5717326402664185,
+      "rewards/margins": 0.8716920018196106,
+      "rewards/rejected": -2.443424701690674,
+      "sft_loss": 1.6421794891357422,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 8.705912983045286,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": -0.14163485169410706,
+      "logits/rejected": -0.04321925342082977,
+      "logps/chosen": -1.607598066329956,
+      "logps/rejected": -2.3242135047912598,
+      "loss": 0.4599,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.607598066329956,
+      "rewards/margins": 0.7166159152984619,
+      "rewards/rejected": -2.3242135047912598,
+      "sft_loss": 1.6503245830535889,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 6.81702621455927,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": -0.2439926117658615,
+      "logits/rejected": -0.05459132790565491,
+      "logps/chosen": -1.5861202478408813,
+      "logps/rejected": -2.08335280418396,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5861202478408813,
+      "rewards/margins": 0.4972327649593353,
+      "rewards/rejected": -2.08335280418396,
+      "sft_loss": 1.6445051431655884,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 8.199519234548399,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": -0.2169719934463501,
+      "logits/rejected": -0.09205500036478043,
+      "logps/chosen": -1.548842430114746,
+      "logps/rejected": -2.2640018463134766,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.548842430114746,
+      "rewards/margins": 0.7151594758033752,
+      "rewards/rejected": -2.2640018463134766,
+      "sft_loss": 1.6609394550323486,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 7.551700105436862,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": -0.3121129274368286,
+      "logits/rejected": -0.1062546968460083,
+      "logps/chosen": -1.6494537591934204,
+      "logps/rejected": -2.213712692260742,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6494537591934204,
+      "rewards/margins": 0.5642590522766113,
+      "rewards/rejected": -2.213712692260742,
+      "sft_loss": 1.6747217178344727,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 11.08626706640028,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": -0.1887563318014145,
+      "logits/rejected": -0.09307411313056946,
+      "logps/chosen": -1.5149710178375244,
+      "logps/rejected": -2.1049327850341797,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5149710178375244,
+      "rewards/margins": 0.5899616479873657,
+      "rewards/rejected": -2.1049327850341797,
+      "sft_loss": 1.585853934288025,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 9.496446162853063,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": -0.14326107501983643,
+      "logits/rejected": -0.17542067170143127,
+      "logps/chosen": -1.4481487274169922,
+      "logps/rejected": -2.071676731109619,
+      "loss": 0.4234,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4481487274169922,
+      "rewards/margins": 0.6235281229019165,
+      "rewards/rejected": -2.071676731109619,
+      "sft_loss": 1.5847502946853638,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 6.652368802603641,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": -0.23582640290260315,
+      "logits/rejected": -0.11037939786911011,
+      "logps/chosen": -1.614134430885315,
+      "logps/rejected": -2.2239456176757812,
+      "loss": 0.4576,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.614134430885315,
+      "rewards/margins": 0.6098113059997559,
+      "rewards/rejected": -2.2239456176757812,
+      "sft_loss": 1.7315566539764404,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 6.233736541983424,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": -0.13952019810676575,
+      "logits/rejected": 0.0490092858672142,
+      "logps/chosen": -1.4858424663543701,
+      "logps/rejected": -2.1162209510803223,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4858424663543701,
+      "rewards/margins": 0.6303783059120178,
+      "rewards/rejected": -2.1162209510803223,
+      "sft_loss": 1.5842537879943848,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 12.749067219519082,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": -0.22309139370918274,
+      "logits/rejected": -0.1758689433336258,
+      "logps/chosen": -1.4894559383392334,
+      "logps/rejected": -2.0888736248016357,
+      "loss": 0.4462,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4894559383392334,
+      "rewards/margins": 0.5994176864624023,
+      "rewards/rejected": -2.0888736248016357,
+      "sft_loss": 1.6486419439315796,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 11.22414600791383,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": -0.30279210209846497,
+      "logits/rejected": -0.09396946430206299,
+      "logps/chosen": -1.50836980342865,
+      "logps/rejected": -2.2401037216186523,
+      "loss": 0.4341,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.50836980342865,
+      "rewards/margins": 0.7317339777946472,
+      "rewards/rejected": -2.2401037216186523,
+      "sft_loss": 1.6456451416015625,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 7.056685347275519,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": -0.21237125992774963,
+      "logits/rejected": -0.08999466896057129,
+      "logps/chosen": -1.5916156768798828,
+      "logps/rejected": -2.1415927410125732,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5916156768798828,
+      "rewards/margins": 0.54997718334198,
+      "rewards/rejected": -2.1415927410125732,
+      "sft_loss": 1.6495697498321533,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 8.831089583385387,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": -0.15597601234912872,
+      "logits/rejected": -0.14214563369750977,
+      "logps/chosen": -1.5390175580978394,
+      "logps/rejected": -2.1770451068878174,
+      "loss": 0.4375,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5390175580978394,
+      "rewards/margins": 0.638027548789978,
+      "rewards/rejected": -2.1770451068878174,
+      "sft_loss": 1.6056034564971924,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 11.814580230373496,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": -0.25163984298706055,
+      "logits/rejected": -0.11963468790054321,
+      "logps/chosen": -1.5341298580169678,
+      "logps/rejected": -2.457284688949585,
+      "loss": 0.409,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5341298580169678,
+      "rewards/margins": 0.9231548309326172,
+      "rewards/rejected": -2.457284688949585,
+      "sft_loss": 1.578685998916626,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 12.25741083867556,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": -0.3177322745323181,
+      "logits/rejected": -0.16120585799217224,
+      "logps/chosen": -1.503438949584961,
+      "logps/rejected": -2.218707323074341,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.503438949584961,
+      "rewards/margins": 0.7152682542800903,
+      "rewards/rejected": -2.218707323074341,
+      "sft_loss": 1.6588207483291626,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 12.822732341972932,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": -0.2096444070339203,
+      "logits/rejected": -0.13994260132312775,
+      "logps/chosen": -1.5128350257873535,
+      "logps/rejected": -2.0648093223571777,
+      "loss": 0.4593,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5128350257873535,
+      "rewards/margins": 0.551974356174469,
+      "rewards/rejected": -2.0648093223571777,
+      "sft_loss": 1.592486023902893,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 10.009567514790708,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": -0.2408403605222702,
+      "logits/rejected": -0.11644059419631958,
+      "logps/chosen": -1.6379438638687134,
+      "logps/rejected": -2.3291876316070557,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6379438638687134,
+      "rewards/margins": 0.6912436485290527,
+      "rewards/rejected": -2.3291876316070557,
+      "sft_loss": 1.751821517944336,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 10.66253837775219,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": -0.2995717525482178,
+      "logits/rejected": -0.05560945346951485,
+      "logps/chosen": -1.6378438472747803,
+      "logps/rejected": -2.179105043411255,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6378438472747803,
+      "rewards/margins": 0.5412610769271851,
+      "rewards/rejected": -2.179105043411255,
+      "sft_loss": 1.6909475326538086,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 8.164331998457351,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": -0.2505989074707031,
+      "logits/rejected": -0.1925095021724701,
+      "logps/chosen": -1.5192012786865234,
+      "logps/rejected": -2.0587570667266846,
+      "loss": 0.485,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5192012786865234,
+      "rewards/margins": 0.539555549621582,
+      "rewards/rejected": -2.0587570667266846,
+      "sft_loss": 1.6563823223114014,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 7.643442799207146,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": -0.36065754294395447,
+      "logits/rejected": -0.19505465030670166,
+      "logps/chosen": -1.5969563722610474,
+      "logps/rejected": -2.221297025680542,
+      "loss": 0.4724,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5969563722610474,
+      "rewards/margins": 0.6243407130241394,
+      "rewards/rejected": -2.221297025680542,
+      "sft_loss": 1.6127878427505493,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 15.705846713940252,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": -0.23518851399421692,
+      "logits/rejected": -0.10746979713439941,
+      "logps/chosen": -1.4499380588531494,
+      "logps/rejected": -2.019010305404663,
+      "loss": 0.4394,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4499380588531494,
+      "rewards/margins": 0.5690725445747375,
+      "rewards/rejected": -2.019010305404663,
+      "sft_loss": 1.5194131135940552,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 10.003051386769481,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": -0.144705131649971,
+      "logits/rejected": -0.019480938091874123,
+      "logps/chosen": -1.602990746498108,
+      "logps/rejected": -2.247408390045166,
+      "loss": 0.4483,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.602990746498108,
+      "rewards/margins": 0.644417941570282,
+      "rewards/rejected": -2.247408390045166,
+      "sft_loss": 1.7288663387298584,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 12.223390468985686,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": -0.24656164646148682,
+      "logits/rejected": -0.019894180819392204,
+      "logps/chosen": -1.6426786184310913,
+      "logps/rejected": -2.375697612762451,
+      "loss": 0.4522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6426786184310913,
+      "rewards/margins": 0.7330188751220703,
+      "rewards/rejected": -2.375697612762451,
+      "sft_loss": 1.6906198263168335,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 8.816260901663746,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": -0.15205910801887512,
+      "logits/rejected": -0.1319962441921234,
+      "logps/chosen": -1.562274694442749,
+      "logps/rejected": -2.0063271522521973,
+      "loss": 0.5216,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.562274694442749,
+      "rewards/margins": 0.4440523684024811,
+      "rewards/rejected": -2.0063271522521973,
+      "sft_loss": 1.6867120265960693,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 7.306427594246043,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": -0.266589879989624,
+      "logits/rejected": -0.19822144508361816,
+      "logps/chosen": -1.5395902395248413,
+      "logps/rejected": -2.1543092727661133,
+      "loss": 0.4559,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5395902395248413,
+      "rewards/margins": 0.6147189140319824,
+      "rewards/rejected": -2.1543092727661133,
+      "sft_loss": 1.6022942066192627,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 16.13599776839836,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": -0.19161126017570496,
+      "logits/rejected": -0.04991975054144859,
+      "logps/chosen": -1.4886987209320068,
+      "logps/rejected": -2.400658130645752,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4886987209320068,
+      "rewards/margins": 0.9119596481323242,
+      "rewards/rejected": -2.400658130645752,
+      "sft_loss": 1.6483510732650757,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 7.869560116788056,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": -0.20632624626159668,
+      "logits/rejected": -0.04783254116773605,
+      "logps/chosen": -1.4557554721832275,
+      "logps/rejected": -2.1471357345581055,
+      "loss": 0.4296,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4557554721832275,
+      "rewards/margins": 0.6913801431655884,
+      "rewards/rejected": -2.1471357345581055,
+      "sft_loss": 1.5793284177780151,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 6.332669004351588,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": -0.3329741656780243,
+      "logits/rejected": -0.05530470609664917,
+      "logps/chosen": -1.46047043800354,
+      "logps/rejected": -2.1376914978027344,
+      "loss": 0.4416,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.46047043800354,
+      "rewards/margins": 0.6772211194038391,
+      "rewards/rejected": -2.1376914978027344,
+      "sft_loss": 1.5921342372894287,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 5.125291681931396,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": -0.27868136763572693,
+      "logits/rejected": -0.09898083657026291,
+      "logps/chosen": -1.59736168384552,
+      "logps/rejected": -2.370673418045044,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.59736168384552,
+      "rewards/margins": 0.7733116149902344,
+      "rewards/rejected": -2.370673418045044,
+      "sft_loss": 1.7875791788101196,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 6.56238996651261,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": -0.19537453353405,
+      "logits/rejected": -0.1279398500919342,
+      "logps/chosen": -1.6379623413085938,
+      "logps/rejected": -2.2381696701049805,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6379623413085938,
+      "rewards/margins": 0.6002073287963867,
+      "rewards/rejected": -2.2381696701049805,
+      "sft_loss": 1.667297124862671,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 9.565512529644018,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": -0.21644096076488495,
+      "logits/rejected": -0.056559689342975616,
+      "logps/chosen": -1.4373853206634521,
+      "logps/rejected": -2.0811617374420166,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4373853206634521,
+      "rewards/margins": 0.6437762975692749,
+      "rewards/rejected": -2.0811617374420166,
+      "sft_loss": 1.5225098133087158,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 8.896316771073908,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": -0.21377773582935333,
+      "logits/rejected": -0.0971156433224678,
+      "logps/chosen": -1.5980726480484009,
+      "logps/rejected": -2.448322057723999,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5980726480484009,
+      "rewards/margins": 0.8502495884895325,
+      "rewards/rejected": -2.448322057723999,
+      "sft_loss": 1.7738758325576782,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": 0.1345709264278412,
+      "eval_logits/rejected": 0.23717787861824036,
+      "eval_logps/chosen": -1.6158757209777832,
+      "eval_logps/rejected": -2.1942226886749268,
+      "eval_loss": 0.4895861744880676,
+      "eval_rewards/accuracies": 0.671364963054657,
+      "eval_rewards/chosen": -1.6158757209777832,
+      "eval_rewards/margins": 0.5783470869064331,
+      "eval_rewards/rejected": -2.1942226886749268,
+      "eval_runtime": 43.0679,
+      "eval_samples_per_second": 31.23,
+      "eval_sft_loss": 1.6913232803344727,
+      "eval_steps_per_second": 7.825,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 0.49661777635968474,
+      "train_runtime": 31536.5493,
+      "train_samples_per_second": 5.688,
+      "train_steps_per_second": 0.178
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}