diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3229 @@
+{
+  "best_metric": 1.3293771743774414,
+  "best_model_checkpoint": "saves/Gemma-2B-It/lora/orpo-salt/checkpoint-1500",
+  "epoch": 2.9969690846635686,
+  "eval_steps": 500,
+  "global_step": 1854,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01616488179430188,
+      "grad_norm": 3.7809898853302,
+      "learning_rate": 4.999648198770648e-06,
+      "logits/chosen": -21.99495506286621,
+      "logits/rejected": -22.17684555053711,
+      "logps/chosen": -2.1064062118530273,
+      "logps/rejected": -2.4076905250549316,
+      "loss": 2.18,
+      "odds_ratio_loss": 0.7356098294258118,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.21064062416553497,
+      "rewards/margins": 0.03012842871248722,
+      "rewards/rejected": -0.24076907336711884,
+      "sft_loss": 2.1064062118530273,
+      "step": 10
+    },
+    {
+      "epoch": 0.03232976358860376,
+      "grad_norm": 2.3069310188293457,
+      "learning_rate": 4.998578646361359e-06,
+      "logits/chosen": -21.845182418823242,
+      "logits/rejected": -21.866207122802734,
+      "logps/chosen": -2.080873489379883,
+      "logps/rejected": -2.124648332595825,
+      "loss": 2.1663,
+      "odds_ratio_loss": 0.8539472818374634,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.20808735489845276,
+      "rewards/margins": 0.004377496428787708,
+      "rewards/rejected": -0.2124648541212082,
+      "sft_loss": 2.080873489379883,
+      "step": 20
+    },
+    {
+      "epoch": 0.04849464538290564,
+      "grad_norm": 5.5587544441223145,
+      "learning_rate": 4.996791614004449e-06,
+      "logits/chosen": -21.90616226196289,
+      "logits/rejected": -21.923837661743164,
+      "logps/chosen": -2.2512574195861816,
+      "logps/rejected": -2.225501537322998,
+      "loss": 2.3454,
+      "odds_ratio_loss": 0.9413110017776489,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.22512571513652802,
+      "rewards/margins": -0.0025755497626960278,
+      "rewards/rejected": -0.2225501835346222,
+      "sft_loss": 2.2512574195861816,
+      "step": 30
+    },
+    {
+      "epoch": 0.06465952717720752,
+      "grad_norm": 3.6638951301574707,
+      "learning_rate": 4.994287614855618e-06,
+      "logits/chosen": -21.935110092163086,
+      "logits/rejected": -22.104324340820312,
+      "logps/chosen": -2.171135425567627,
+      "logps/rejected": -2.2752678394317627,
+      "loss": 2.267,
+      "odds_ratio_loss": 0.9586257934570312,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.21711353957653046,
+      "rewards/margins": 0.010413245297968388,
+      "rewards/rejected": -0.22752678394317627,
+      "sft_loss": 2.171135425567627,
+      "step": 40
+    },
+    {
+      "epoch": 0.0808244089715094,
+      "grad_norm": 6.454894065856934,
+      "learning_rate": 4.991067367951343e-06,
+      "logits/chosen": -22.541332244873047,
+      "logits/rejected": -22.472043991088867,
+      "logps/chosen": -2.1005358695983887,
+      "logps/rejected": -2.1757073402404785,
+      "loss": 2.1825,
+      "odds_ratio_loss": 0.8198438882827759,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.21005356311798096,
+      "rewards/margins": 0.007517138961702585,
+      "rewards/rejected": -0.2175707072019577,
+      "sft_loss": 2.1005358695983887,
+      "step": 50
+    },
+    {
+      "epoch": 0.09698929076581128,
+      "grad_norm": 2.4648330211639404,
+      "learning_rate": 4.987131798002389e-06,
+      "logits/chosen": -22.33713722229004,
+      "logits/rejected": -22.142881393432617,
+      "logps/chosen": -2.0364699363708496,
+      "logps/rejected": -2.22188138961792,
+      "loss": 2.1252,
+      "odds_ratio_loss": 0.887610912322998,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.20364701747894287,
+      "rewards/margins": 0.018541133031249046,
+      "rewards/rejected": -0.22218814492225647,
+      "sft_loss": 2.0364699363708496,
+      "step": 60
+    },
+    {
+      "epoch": 0.11315417256011315,
+      "grad_norm": 4.808262825012207,
+      "learning_rate": 4.982482035128285e-06,
+      "logits/chosen": -22.240699768066406,
+      "logits/rejected": -22.348970413208008,
+      "logps/chosen": -2.144533634185791,
+      "logps/rejected": -2.3089094161987305,
+      "loss": 2.2393,
+      "odds_ratio_loss": 0.9479582905769348,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.21445336937904358,
+      "rewards/margins": 0.01643756404519081,
+      "rewards/rejected": -0.23089094460010529,
+      "sft_loss": 2.144533634185791,
+      "step": 70
+    },
+    {
+      "epoch": 0.12931905435441504,
+      "grad_norm": 5.7843122482299805,
+      "learning_rate": 4.9771194145328e-06,
+      "logits/chosen": -22.19432258605957,
+      "logits/rejected": -22.165550231933594,
+      "logps/chosen": -1.7035343647003174,
+      "logps/rejected": -2.021710157394409,
+      "loss": 1.7739,
+      "odds_ratio_loss": 0.7040209174156189,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.17035342752933502,
+      "rewards/margins": 0.03181760013103485,
+      "rewards/rejected": -0.20217104256153107,
+      "sft_loss": 1.7035343647003174,
+      "step": 80
+    },
+    {
+      "epoch": 0.1454839361487169,
+      "grad_norm": 10.499777793884277,
+      "learning_rate": 4.971045476120532e-06,
+      "logits/chosen": -22.63768196105957,
+      "logits/rejected": -22.5053768157959,
+      "logps/chosen": -1.7764383554458618,
+      "logps/rejected": -1.8638198375701904,
+      "loss": 1.8559,
+      "odds_ratio_loss": 0.7946864366531372,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.17764383554458618,
+      "rewards/margins": 0.008738175965845585,
+      "rewards/rejected": -0.1863820105791092,
+      "sft_loss": 1.7764383554458618,
+      "step": 90
+    },
+    {
+      "epoch": 0.1616488179430188,
+      "grad_norm": 2.4865188598632812,
+      "learning_rate": 4.964261964054713e-06,
+      "logits/chosen": -22.539587020874023,
+      "logits/rejected": -22.56546401977539,
+      "logps/chosen": -1.846840262413025,
+      "logps/rejected": -2.0120983123779297,
+      "loss": 1.9298,
+      "odds_ratio_loss": 0.829124927520752,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.18468405306339264,
+      "rewards/margins": 0.016525788232684135,
+      "rewards/rejected": -0.20120985805988312,
+      "sft_loss": 1.846840262413025,
+      "step": 100
+    },
+    {
+      "epoch": 0.17781369973732067,
+      "grad_norm": 8.431103706359863,
+      "learning_rate": 4.956770826256372e-06,
+      "logits/chosen": -22.575443267822266,
+      "logits/rejected": -22.463134765625,
+      "logps/chosen": -1.6286426782608032,
+      "logps/rejected": -1.7305223941802979,
+      "loss": 1.7064,
+      "odds_ratio_loss": 0.7774398922920227,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.16286428272724152,
+      "rewards/margins": 0.010187968611717224,
+      "rewards/rejected": -0.17305223643779755,
+      "sft_loss": 1.6286426782608032,
+      "step": 110
+    },
+    {
+      "epoch": 0.19397858153162256,
+      "grad_norm": 2.32936429977417,
+      "learning_rate": 4.94857421384497e-06,
+      "logits/chosen": -22.672468185424805,
+      "logits/rejected": -22.749752044677734,
+      "logps/chosen": -1.6404567956924438,
+      "logps/rejected": -1.8715522289276123,
+      "loss": 1.7191,
+      "odds_ratio_loss": 0.7863814830780029,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.16404569149017334,
+      "rewards/margins": 0.023109545931220055,
+      "rewards/rejected": -0.18715524673461914,
+      "sft_loss": 1.6404567956924438,
+      "step": 120
+    },
+    {
+      "epoch": 0.21014346332592443,
+      "grad_norm": 2.062532663345337,
+      "learning_rate": 4.939674480520701e-06,
+      "logits/chosen": -22.727245330810547,
+      "logits/rejected": -22.860414505004883,
+      "logps/chosen": -1.5943523645401,
+      "logps/rejected": -1.7242279052734375,
+      "loss": 1.6786,
+      "odds_ratio_loss": 0.8422672152519226,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1594352275133133,
+      "rewards/margins": 0.012987568974494934,
+      "rewards/rejected": -0.17242279648780823,
+      "sft_loss": 1.5943523645401,
+      "step": 130
+    },
+    {
+      "epoch": 0.2263083451202263,
+      "grad_norm": 1.7461190223693848,
+      "learning_rate": 4.930074181888613e-06,
+      "logits/chosen": -22.773456573486328,
+      "logits/rejected": -22.877899169921875,
+      "logps/chosen": -1.680092453956604,
+      "logps/rejected": -1.8227007389068604,
+      "loss": 1.7527,
+      "odds_ratio_loss": 0.7260502576828003,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.16800925135612488,
+      "rewards/margins": 0.014260831288993359,
+      "rewards/rejected": -0.1822700798511505,
+      "sft_loss": 1.680092453956604,
+      "step": 140
+    },
+    {
+      "epoch": 0.2424732269145282,
+      "grad_norm": 3.3798775672912598,
+      "learning_rate": 4.91977607472475e-06,
+      "logits/chosen": -23.01150131225586,
+      "logits/rejected": -22.988527297973633,
+      "logps/chosen": -1.5780761241912842,
+      "logps/rejected": -1.6528291702270508,
+      "loss": 1.6563,
+      "odds_ratio_loss": 0.7826321721076965,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1578076183795929,
+      "rewards/margins": 0.007475291378796101,
+      "rewards/rejected": -0.16528290510177612,
+      "sft_loss": 1.5780761241912842,
+      "step": 150
+    },
+    {
+      "epoch": 0.2586381087088301,
+      "grad_norm": 1.8430626392364502,
+      "learning_rate": 4.908783116184534e-06,
+      "logits/chosen": -22.70990562438965,
+      "logits/rejected": -22.700252532958984,
+      "logps/chosen": -1.5454621315002441,
+      "logps/rejected": -1.7210956811904907,
+      "loss": 1.614,
+      "odds_ratio_loss": 0.6857101917266846,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.15454623103141785,
+      "rewards/margins": 0.017563361674547195,
+      "rewards/rejected": -0.17210958898067474,
+      "sft_loss": 1.5454621315002441,
+      "step": 160
+    },
+    {
+      "epoch": 0.27480299050313195,
+      "grad_norm": 1.635880470275879,
+      "learning_rate": 4.897098462953598e-06,
+      "logits/chosen": -23.044336318969727,
+      "logits/rejected": -23.024044036865234,
+      "logps/chosen": -1.457194447517395,
+      "logps/rejected": -1.7030904293060303,
+      "loss": 1.5297,
+      "odds_ratio_loss": 0.7254581451416016,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.14571943879127502,
+      "rewards/margins": 0.02458963170647621,
+      "rewards/rejected": -0.17030906677246094,
+      "sft_loss": 1.457194447517395,
+      "step": 170
+    },
+    {
+      "epoch": 0.2909678722974338,
+      "grad_norm": 1.3249585628509521,
+      "learning_rate": 4.884725470341331e-06,
+      "logits/chosen": -23.00472640991211,
+      "logits/rejected": -23.162250518798828,
+      "logps/chosen": -1.3314763307571411,
+      "logps/rejected": -1.5617011785507202,
+      "loss": 1.3961,
+      "odds_ratio_loss": 0.6465403437614441,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.13314762711524963,
+      "rewards/margins": 0.023022485896945,
+      "rewards/rejected": -0.15617012977600098,
+      "sft_loss": 1.3314763307571411,
+      "step": 180
+    },
+    {
+      "epoch": 0.3071327540917357,
+      "grad_norm": 4.37128210067749,
+      "learning_rate": 4.871667691317377e-06,
+      "logits/chosen": -23.25027847290039,
+      "logits/rejected": -23.159801483154297,
+      "logps/chosen": -1.5023282766342163,
+      "logps/rejected": -1.5170981884002686,
+      "loss": 1.5883,
+      "odds_ratio_loss": 0.8601799011230469,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.15023282170295715,
+      "rewards/margins": 0.0014769884292036295,
+      "rewards/rejected": -0.15170980989933014,
+      "sft_loss": 1.5023282766342163,
+      "step": 190
+    },
+    {
+      "epoch": 0.3232976358860376,
+      "grad_norm": 12.613110542297363,
+      "learning_rate": 4.857928875491392e-06,
+      "logits/chosen": -23.007986068725586,
+      "logits/rejected": -22.91147232055664,
+      "logps/chosen": -1.360762596130371,
+      "logps/rejected": -1.479021668434143,
+      "loss": 1.4383,
+      "odds_ratio_loss": 0.7751696705818176,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13607625663280487,
+      "rewards/margins": 0.01182589866220951,
+      "rewards/rejected": -0.14790216088294983,
+      "sft_loss": 1.360762596130371,
+      "step": 200
+    },
+    {
+      "epoch": 0.33946251768033947,
+      "grad_norm": 2.2189383506774902,
+      "learning_rate": 4.843512968036314e-06,
+      "logits/chosen": -22.693614959716797,
+      "logits/rejected": -22.657732009887695,
+      "logps/chosen": -1.3642535209655762,
+      "logps/rejected": -1.4226882457733154,
+      "loss": 1.4427,
+      "odds_ratio_loss": 0.7848214507102966,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.13642534613609314,
+      "rewards/margins": 0.0058434633538126945,
+      "rewards/rejected": -0.1422688066959381,
+      "sft_loss": 1.3642535209655762,
+      "step": 210
+    },
+    {
+      "epoch": 0.35562739947464134,
+      "grad_norm": 4.814913749694824,
+      "learning_rate": 4.828424108555486e-06,
+      "logits/chosen": -23.161609649658203,
+      "logits/rejected": -23.108890533447266,
+      "logps/chosen": -1.5788774490356445,
+      "logps/rejected": -1.7273142337799072,
+      "loss": 1.6537,
+      "odds_ratio_loss": 0.748489499092102,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.15788774192333221,
+      "rewards/margins": 0.014843670651316643,
+      "rewards/rejected": -0.1727314293384552,
+      "sft_loss": 1.5788774490356445,
+      "step": 220
+    },
+    {
+      "epoch": 0.3717922812689432,
+      "grad_norm": 0.8914692997932434,
+      "learning_rate": 4.812666629893957e-06,
+      "logits/chosen": -23.069910049438477,
+      "logits/rejected": -23.09903907775879,
+      "logps/chosen": -1.3772187232971191,
+      "logps/rejected": -1.4173983335494995,
+      "loss": 1.4523,
+      "odds_ratio_loss": 0.7508037686347961,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13772186636924744,
+      "rewards/margins": 0.004017965402454138,
+      "rewards/rejected": -0.1417398303747177,
+      "sft_loss": 1.3772187232971191,
+      "step": 230
+    },
+    {
+      "epoch": 0.3879571630632451,
+      "grad_norm": 1.202868103981018,
+      "learning_rate": 4.796245056894273e-06,
+      "logits/chosen": -22.570789337158203,
+      "logits/rejected": -22.600833892822266,
+      "logps/chosen": -1.453567385673523,
+      "logps/rejected": -1.5844666957855225,
+      "loss": 1.5321,
+      "odds_ratio_loss": 0.7851130962371826,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14535674452781677,
+      "rewards/margins": 0.013089925050735474,
+      "rewards/rejected": -0.15844666957855225,
+      "sft_loss": 1.453567385673523,
+      "step": 240
+    },
+    {
+      "epoch": 0.404122044857547,
+      "grad_norm": 1.3130539655685425,
+      "learning_rate": 4.779164105097148e-06,
+      "logits/chosen": -23.004568099975586,
+      "logits/rejected": -23.11286163330078,
+      "logps/chosen": -1.325037956237793,
+      "logps/rejected": -1.563007116317749,
+      "loss": 1.391,
+      "odds_ratio_loss": 0.6598650813102722,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13250377774238586,
+      "rewards/margins": 0.023796914145350456,
+      "rewards/rejected": -0.15630069375038147,
+      "sft_loss": 1.325037956237793,
+      "step": 250
+    },
+    {
+      "epoch": 0.42028692665184886,
+      "grad_norm": 1.867422342300415,
+      "learning_rate": 4.761428679387373e-06,
+      "logits/chosen": -22.629825592041016,
+      "logits/rejected": -22.818918228149414,
+      "logps/chosen": -1.317091941833496,
+      "logps/rejected": -1.4807102680206299,
+      "loss": 1.3842,
+      "odds_ratio_loss": 0.6714745759963989,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.13170920312404633,
+      "rewards/margins": 0.016361821442842484,
+      "rewards/rejected": -0.1480710208415985,
+      "sft_loss": 1.317091941833496,
+      "step": 260
+    },
+    {
+      "epoch": 0.4364518084461507,
+      "grad_norm": 1.9516927003860474,
+      "learning_rate": 4.7430438725853515e-06,
+      "logits/chosen": -22.51862907409668,
+      "logits/rejected": -22.682025909423828,
+      "logps/chosen": -1.4145148992538452,
+      "logps/rejected": -1.653055191040039,
+      "loss": 1.4844,
+      "odds_ratio_loss": 0.6991928815841675,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.14145150780677795,
+      "rewards/margins": 0.023854022845625877,
+      "rewards/rejected": -0.16530552506446838,
+      "sft_loss": 1.4145148992538452,
+      "step": 270
+    },
+    {
+      "epoch": 0.4526166902404526,
+      "grad_norm": 1.7087641954421997,
+      "learning_rate": 4.724014963984669e-06,
+      "logits/chosen": -22.849842071533203,
+      "logits/rejected": -22.771404266357422,
+      "logps/chosen": -1.387978196144104,
+      "logps/rejected": -1.5646307468414307,
+      "loss": 1.4589,
+      "odds_ratio_loss": 0.7090586423873901,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.1387978196144104,
+      "rewards/margins": 0.01766524836421013,
+      "rewards/rejected": -0.15646305680274963,
+      "sft_loss": 1.387978196144104,
+      "step": 280
+    },
+    {
+      "epoch": 0.4687815720347545,
+      "grad_norm": 3.348365068435669,
+      "learning_rate": 4.704347417836116e-06,
+      "logits/chosen": -22.53603172302246,
+      "logits/rejected": -22.64337730407715,
+      "logps/chosen": -1.2867505550384521,
+      "logps/rejected": -1.4700194597244263,
+      "loss": 1.3571,
+      "odds_ratio_loss": 0.7038428783416748,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.12867505848407745,
+      "rewards/margins": 0.018326906487345695,
+      "rewards/rejected": -0.1470019519329071,
+      "sft_loss": 1.2867505550384521,
+      "step": 290
+    },
+    {
+      "epoch": 0.4849464538290564,
+      "grad_norm": 2.8705925941467285,
+      "learning_rate": 4.684046881778603e-06,
+      "logits/chosen": -22.37331771850586,
+      "logits/rejected": -22.327777862548828,
+      "logps/chosen": -1.3365970849990845,
+      "logps/rejected": -1.4279487133026123,
+      "loss": 1.4075,
+      "odds_ratio_loss": 0.7086235880851746,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13365969061851501,
+      "rewards/margins": 0.009135158732533455,
+      "rewards/rejected": -0.14279486238956451,
+      "sft_loss": 1.3365970849990845,
+      "step": 300
+    },
+    {
+      "epoch": 0.5011113356233583,
+      "grad_norm": 1.1998392343521118,
+      "learning_rate": 4.663119185217409e-06,
+      "logits/chosen": -22.524127960205078,
+      "logits/rejected": -22.61183738708496,
+      "logps/chosen": -1.3241046667099,
+      "logps/rejected": -1.48367440700531,
+      "loss": 1.3928,
+      "odds_ratio_loss": 0.6864575147628784,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.13241048157215118,
+      "rewards/margins": 0.015956982970237732,
+      "rewards/rejected": -0.14836746454238892,
+      "sft_loss": 1.3241046667099,
+      "step": 310
+    },
+    {
+      "epoch": 0.5172762174176602,
+      "grad_norm": 1.1742535829544067,
+      "learning_rate": 4.641570337650232e-06,
+      "logits/chosen": -22.668704986572266,
+      "logits/rejected": -22.804088592529297,
+      "logps/chosen": -1.2428923845291138,
+      "logps/rejected": -1.3876885175704956,
+      "loss": 1.3138,
+      "odds_ratio_loss": 0.7085933685302734,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12428925186395645,
+      "rewards/margins": 0.014479607343673706,
+      "rewards/rejected": -0.13876885175704956,
+      "sft_loss": 1.2428923845291138,
+      "step": 320
+    },
+    {
+      "epoch": 0.533441099211962,
+      "grad_norm": 3.9174962043762207,
+      "learning_rate": 4.61940652694154e-06,
+      "logits/chosen": -22.036165237426758,
+      "logits/rejected": -22.13457679748535,
+      "logps/chosen": -1.3345186710357666,
+      "logps/rejected": -1.4239206314086914,
+      "loss": 1.4082,
+      "odds_ratio_loss": 0.7365013360977173,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.13345186412334442,
+      "rewards/margins": 0.008940198458731174,
+      "rewards/rejected": -0.14239206910133362,
+      "sft_loss": 1.3345186710357666,
+      "step": 330
+    },
+    {
+      "epoch": 0.5496059810062639,
+      "grad_norm": 3.5989551544189453,
+      "learning_rate": 4.596634117545689e-06,
+      "logits/chosen": -22.659748077392578,
+      "logits/rejected": -22.487016677856445,
+      "logps/chosen": -1.4412176609039307,
+      "logps/rejected": -1.6016347408294678,
+      "loss": 1.5117,
+      "odds_ratio_loss": 0.7047333121299744,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.14412176609039307,
+      "rewards/margins": 0.016041725873947144,
+      "rewards/rejected": -0.1601634919643402,
+      "sft_loss": 1.4412176609039307,
+      "step": 340
+    },
+    {
+      "epoch": 0.5657708628005658,
+      "grad_norm": 1.72525954246521,
+      "learning_rate": 4.573259648679335e-06,
+      "logits/chosen": -22.422473907470703,
+      "logits/rejected": -22.158262252807617,
+      "logps/chosen": -1.3891839981079102,
+      "logps/rejected": -1.5595767498016357,
+      "loss": 1.4614,
+      "odds_ratio_loss": 0.7222477197647095,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.13891839981079102,
+      "rewards/margins": 0.017039284110069275,
+      "rewards/rejected": -0.1559576839208603,
+      "sft_loss": 1.3891839981079102,
+      "step": 350
+    },
+    {
+      "epoch": 0.5819357445948676,
+      "grad_norm": 6.434234619140625,
+      "learning_rate": 4.549289832443663e-06,
+      "logits/chosen": -22.174190521240234,
+      "logits/rejected": -22.23097038269043,
+      "logps/chosen": -1.315002202987671,
+      "logps/rejected": -1.472235918045044,
+      "loss": 1.3873,
+      "odds_ratio_loss": 0.7233031988143921,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1315002143383026,
+      "rewards/margins": 0.015723396092653275,
+      "rewards/rejected": -0.14722362160682678,
+      "sft_loss": 1.315002202987671,
+      "step": 360
+    },
+    {
+      "epoch": 0.5981006263891695,
+      "grad_norm": 1.595171332359314,
+      "learning_rate": 4.524731551896978e-06,
+      "logits/chosen": -21.987152099609375,
+      "logits/rejected": -21.909650802612305,
+      "logps/chosen": -1.2452141046524048,
+      "logps/rejected": -1.389278769493103,
+      "loss": 1.3152,
+      "odds_ratio_loss": 0.7003225088119507,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1245214194059372,
+      "rewards/margins": 0.014406457543373108,
+      "rewards/rejected": -0.1389278769493103,
+      "sft_loss": 1.2452141046524048,
+      "step": 370
+    },
+    {
+      "epoch": 0.6142655081834714,
+      "grad_norm": 2.505262851715088,
+      "learning_rate": 4.4995918590781925e-06,
+      "logits/chosen": -22.261516571044922,
+      "logits/rejected": -22.25895118713379,
+      "logps/chosen": -1.3019020557403564,
+      "logps/rejected": -1.4299681186676025,
+      "loss": 1.3751,
+      "odds_ratio_loss": 0.7319445610046387,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.13019022345542908,
+      "rewards/margins": 0.012806607410311699,
+      "rewards/rejected": -0.14299681782722473,
+      "sft_loss": 1.3019020557403564,
+      "step": 380
+    },
+    {
+      "epoch": 0.6304303899777733,
+      "grad_norm": 1.0513848066329956,
+      "learning_rate": 4.473877972981797e-06,
+      "logits/chosen": -21.957128524780273,
+      "logits/rejected": -21.8829288482666,
+      "logps/chosen": -1.343338131904602,
+      "logps/rejected": -1.5255149602890015,
+      "loss": 1.4114,
+      "odds_ratio_loss": 0.6804416179656982,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.13433381915092468,
+      "rewards/margins": 0.01821768842637539,
+      "rewards/rejected": -0.15255150198936462,
+      "sft_loss": 1.343338131904602,
+      "step": 390
+    },
+    {
+      "epoch": 0.6465952717720752,
+      "grad_norm": 2.7681405544281006,
+      "learning_rate": 4.447597277484894e-06,
+      "logits/chosen": -21.926578521728516,
+      "logits/rejected": -21.960134506225586,
+      "logps/chosen": -1.2327673435211182,
+      "logps/rejected": -1.3560773134231567,
+      "loss": 1.305,
+      "odds_ratio_loss": 0.7222930192947388,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.1232767328619957,
+      "rewards/margins": 0.012331007048487663,
+      "rewards/rejected": -0.1356077492237091,
+      "sft_loss": 1.2327673435211182,
+      "step": 400
+    },
+    {
+      "epoch": 0.6627601535663771,
+      "grad_norm": 1.4995496273040771,
+      "learning_rate": 4.42075731922687e-06,
+      "logits/chosen": -22.4599552154541,
+      "logits/rejected": -22.27519989013672,
+      "logps/chosen": -1.383239507675171,
+      "logps/rejected": -1.4447576999664307,
+      "loss": 1.4576,
+      "odds_ratio_loss": 0.7433907389640808,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13832393288612366,
+      "rewards/margins": 0.006151811219751835,
+      "rewards/rejected": -0.1444757580757141,
+      "sft_loss": 1.383239507675171,
+      "step": 410
+    },
+    {
+      "epoch": 0.6789250353606789,
+      "grad_norm": 3.4521114826202393,
+      "learning_rate": 4.3933658054423465e-06,
+      "logits/chosen": -22.025806427001953,
+      "logits/rejected": -21.983091354370117,
+      "logps/chosen": -1.2636308670043945,
+      "logps/rejected": -1.3890178203582764,
+      "loss": 1.3327,
+      "odds_ratio_loss": 0.6908599138259888,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12636308372020721,
+      "rewards/margins": 0.012538706883788109,
+      "rewards/rejected": -0.13890179991722107,
+      "sft_loss": 1.2636308670043945,
+      "step": 420
+    },
+    {
+      "epoch": 0.6950899171549808,
+      "grad_norm": 2.2511396408081055,
+      "learning_rate": 4.365430601748003e-06,
+      "logits/chosen": -21.880239486694336,
+      "logits/rejected": -22.0391845703125,
+      "logps/chosen": -1.4012739658355713,
+      "logps/rejected": -1.4651644229888916,
+      "loss": 1.4754,
+      "odds_ratio_loss": 0.7415169477462769,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.14012740552425385,
+      "rewards/margins": 0.006389034446328878,
+      "rewards/rejected": -0.14651645720005035,
+      "sft_loss": 1.4012739658355713,
+      "step": 430
+    },
+    {
+      "epoch": 0.7112547989492827,
+      "grad_norm": 6.550912857055664,
+      "learning_rate": 4.336959729883925e-06,
+      "logits/chosen": -22.025110244750977,
+      "logits/rejected": -21.944046020507812,
+      "logps/chosen": -1.2905068397521973,
+      "logps/rejected": -1.3223841190338135,
+      "loss": 1.3664,
+      "odds_ratio_loss": 0.7585982084274292,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.12905070185661316,
+      "rewards/margins": 0.003187725320458412,
+      "rewards/rejected": -0.13223841786384583,
+      "sft_loss": 1.2905068397521973,
+      "step": 440
+    },
+    {
+      "epoch": 0.7274196807435845,
+      "grad_norm": 1.741584062576294,
+      "learning_rate": 4.307961365410118e-06,
+      "logits/chosen": -21.6479434967041,
+      "logits/rejected": -21.64910888671875,
+      "logps/chosen": -1.3282034397125244,
+      "logps/rejected": -1.3957862854003906,
+      "loss": 1.4023,
+      "odds_ratio_loss": 0.7411014437675476,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.13282035291194916,
+      "rewards/margins": 0.006758278701454401,
+      "rewards/rejected": -0.13957861065864563,
+      "sft_loss": 1.3282034397125244,
+      "step": 450
+    },
+    {
+      "epoch": 0.7435845625378864,
+      "grad_norm": 2.630305528640747,
+      "learning_rate": 4.278443835358854e-06,
+      "logits/chosen": -22.009098052978516,
+      "logits/rejected": -21.944019317626953,
+      "logps/chosen": -1.2518932819366455,
+      "logps/rejected": -1.4686853885650635,
+      "loss": 1.3164,
+      "odds_ratio_loss": 0.6453157067298889,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12518933415412903,
+      "rewards/margins": 0.0216792281717062,
+      "rewards/rejected": -0.14686855673789978,
+      "sft_loss": 1.2518932819366455,
+      "step": 460
+    },
+    {
+      "epoch": 0.7597494443321883,
+      "grad_norm": 1.7374227046966553,
+      "learning_rate": 4.248415615843523e-06,
+      "logits/chosen": -22.067617416381836,
+      "logits/rejected": -21.912487030029297,
+      "logps/chosen": -1.278611421585083,
+      "logps/rejected": -1.367891550064087,
+      "loss": 1.3508,
+      "odds_ratio_loss": 0.7223629951477051,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1278611272573471,
+      "rewards/margins": 0.008928016759455204,
+      "rewards/rejected": -0.13678915798664093,
+      "sft_loss": 1.278611421585083,
+      "step": 470
+    },
+    {
+      "epoch": 0.7759143261264903,
+      "grad_norm": 5.568856716156006,
+      "learning_rate": 4.217885329624666e-06,
+      "logits/chosen": -22.169946670532227,
+      "logits/rejected": -22.093523025512695,
+      "logps/chosen": -1.207793951034546,
+      "logps/rejected": -1.4276247024536133,
+      "loss": 1.2738,
+      "odds_ratio_loss": 0.660278856754303,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1207793727517128,
+      "rewards/margins": 0.021983083337545395,
+      "rewards/rejected": -0.1427624672651291,
+      "sft_loss": 1.207793951034546,
+      "step": 480
+    },
+    {
+      "epoch": 0.7920792079207921,
+      "grad_norm": 1.7587084770202637,
+      "learning_rate": 4.186861743633911e-06,
+      "logits/chosen": -21.444040298461914,
+      "logits/rejected": -21.620454788208008,
+      "logps/chosen": -1.2591201066970825,
+      "logps/rejected": -1.494167685508728,
+      "loss": 1.3277,
+      "odds_ratio_loss": 0.6855591535568237,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.12591201066970825,
+      "rewards/margins": 0.023504754528403282,
+      "rewards/rejected": -0.14941677451133728,
+      "sft_loss": 1.2591201066970825,
+      "step": 490
+    },
+    {
+      "epoch": 0.808244089715094,
+      "grad_norm": 1.509691596031189,
+      "learning_rate": 4.155353766456497e-06,
+      "logits/chosen": -22.305217742919922,
+      "logits/rejected": -22.103668212890625,
+      "logps/chosen": -1.335773229598999,
+      "logps/rejected": -1.4481773376464844,
+      "loss": 1.4057,
+      "odds_ratio_loss": 0.6996302604675293,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.13357731699943542,
+      "rewards/margins": 0.011240402236580849,
+      "rewards/rejected": -0.14481773972511292,
+      "sft_loss": 1.335773229598999,
+      "step": 500
+    },
+    {
+      "epoch": 0.808244089715094,
+      "eval_logits/chosen": -21.914384841918945,
+      "eval_logits/rejected": -21.94023323059082,
+      "eval_logps/chosen": -1.31483793258667,
+      "eval_logps/rejected": -1.4522359371185303,
+      "eval_loss": 1.38706636428833,
+      "eval_odds_ratio_loss": 0.7222831845283508,
+      "eval_rewards/accuracies": 0.5318182110786438,
+      "eval_rewards/chosen": -0.131483793258667,
+      "eval_rewards/margins": 0.01373977866023779,
+      "eval_rewards/rejected": -0.14522357285022736,
+      "eval_runtime": 82.3592,
+      "eval_samples_per_second": 13.356,
+      "eval_sft_loss": 1.31483793258667,
+      "eval_steps_per_second": 6.678,
+      "step": 500
+    },
+    {
+      "epoch": 0.8244089715093958,
+      "grad_norm": 1.657311201095581,
+      "learning_rate": 4.123370445773134e-06,
+      "logits/chosen": -22.07869529724121,
+      "logits/rejected": -22.091659545898438,
+      "logps/chosen": -1.2787725925445557,
+      "logps/rejected": -1.3090568780899048,
+      "loss": 1.3519,
+      "odds_ratio_loss": 0.731525719165802,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12787726521492004,
+      "rewards/margins": 0.003028427716344595,
+      "rewards/rejected": -0.13090567290782928,
+      "sft_loss": 1.2787725925445557,
+      "step": 510
+    },
+    {
+      "epoch": 0.8405738533036977,
+      "grad_norm": 4.675573825836182,
+      "learning_rate": 4.090920965761906e-06,
+      "logits/chosen": -21.979219436645508,
+      "logits/rejected": -21.770206451416016,
+      "logps/chosen": -1.304009199142456,
+      "logps/rejected": -1.4551854133605957,
+      "loss": 1.3752,
+      "odds_ratio_loss": 0.7114746570587158,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13040092587471008,
+      "rewards/margins": 0.015117612667381763,
+      "rewards/rejected": -0.14551854133605957,
+      "sft_loss": 1.304009199142456,
+      "step": 520
+    },
+    {
+      "epoch": 0.8567387350979996,
+      "grad_norm": 4.995480060577393,
+      "learning_rate": 4.058014644460991e-06,
+      "logits/chosen": -21.77149772644043,
+      "logits/rejected": -21.79526710510254,
+      "logps/chosen": -1.3110116720199585,
+      "logps/rejected": -1.4032762050628662,
+      "loss": 1.3819,
+      "odds_ratio_loss": 0.708578884601593,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13110119104385376,
+      "rewards/margins": 0.009226455353200436,
+      "rewards/rejected": -0.14032763242721558,
+      "sft_loss": 1.3110116720199585,
+      "step": 530
+    },
+    {
+      "epoch": 0.8729036168923014,
+      "grad_norm": 1.891120433807373,
+      "learning_rate": 4.024660931092939e-06,
+      "logits/chosen": -21.345439910888672,
+      "logits/rejected": -21.40594482421875,
+      "logps/chosen": -1.3298126459121704,
+      "logps/rejected": -1.4730676412582397,
+      "loss": 1.4012,
+      "odds_ratio_loss": 0.713805079460144,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13298127055168152,
+      "rewards/margins": 0.014325493946671486,
+      "rewards/rejected": -0.14730675518512726,
+      "sft_loss": 1.3298126459121704,
+      "step": 540
+    },
+    {
+      "epoch": 0.8890684986866033,
+      "grad_norm": 7.624050617218018,
+      "learning_rate": 3.990869403351272e-06,
+      "logits/chosen": -21.680253982543945,
+      "logits/rejected": -21.578588485717773,
+      "logps/chosen": -1.3242005109786987,
+      "logps/rejected": -1.496756911277771,
+      "loss": 1.3887,
+      "odds_ratio_loss": 0.644872784614563,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.13242004811763763,
+      "rewards/margins": 0.017255645245313644,
+      "rewards/rejected": -0.14967571198940277,
+      "sft_loss": 1.3242005109786987,
+      "step": 550
+    },
+    {
+      "epoch": 0.9052333804809052,
+      "grad_norm": 2.1821932792663574,
+      "learning_rate": 3.956649764650206e-06,
+      "logits/chosen": -22.087711334228516,
+      "logits/rejected": -22.087757110595703,
+      "logps/chosen": -1.2911579608917236,
+      "logps/rejected": -1.4688221216201782,
+      "loss": 1.3633,
+      "odds_ratio_loss": 0.7218787670135498,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.12911580502986908,
+      "rewards/margins": 0.017766421660780907,
+      "rewards/rejected": -0.14688222110271454,
+      "sft_loss": 1.2911579608917236,
+      "step": 560
+    },
+    {
+      "epoch": 0.9213982622752072,
+      "grad_norm": 5.107442855834961,
+      "learning_rate": 3.92201184133826e-06,
+      "logits/chosen": -21.773061752319336,
+      "logits/rejected": -21.753795623779297,
+      "logps/chosen": -1.2945088148117065,
+      "logps/rejected": -1.4896302223205566,
+      "loss": 1.3613,
+      "odds_ratio_loss": 0.6681925654411316,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12945088744163513,
+      "rewards/margins": 0.019512150436639786,
+      "rewards/rejected": -0.14896303415298462,
+      "sft_loss": 1.2945088148117065,
+      "step": 570
+    },
+    {
+      "epoch": 0.937563144069509,
+      "grad_norm": 1.4870102405548096,
+      "learning_rate": 3.886965579876572e-06,
+      "logits/chosen": -21.675262451171875,
+      "logits/rejected": -21.543224334716797,
+      "logps/chosen": -1.3047420978546143,
+      "logps/rejected": -1.3921304941177368,
+      "loss": 1.3741,
+      "odds_ratio_loss": 0.694009006023407,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.1304742395877838,
+      "rewards/margins": 0.008738822303712368,
+      "rewards/rejected": -0.13921304047107697,
+      "sft_loss": 1.3047420978546143,
+      "step": 580
+    },
+    {
+      "epoch": 0.9537280258638109,
+      "grad_norm": 2.04402494430542,
+      "learning_rate": 3.851521043982716e-06,
+      "logits/chosen": -21.711593627929688,
+      "logits/rejected": -21.78032875061035,
+      "logps/chosen": -1.311596155166626,
+      "logps/rejected": -1.351053237915039,
+      "loss": 1.3862,
+      "odds_ratio_loss": 0.7462160587310791,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.13115961849689484,
+      "rewards/margins": 0.00394571665674448,
+      "rewards/rejected": -0.13510534167289734,
+      "sft_loss": 1.311596155166626,
+      "step": 590
+    },
+    {
+      "epoch": 0.9698929076581128,
+      "grad_norm": 8.317533493041992,
+      "learning_rate": 3.81568841174086e-06,
+      "logits/chosen": -21.358070373535156,
+      "logits/rejected": -21.403278350830078,
+      "logps/chosen": -1.361669898033142,
+      "logps/rejected": -1.494240403175354,
+      "loss": 1.4338,
+      "odds_ratio_loss": 0.7209025025367737,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1361669898033142,
+      "rewards/margins": 0.013257039710879326,
+      "rewards/rejected": -0.14942404627799988,
+      "sft_loss": 1.361669898033142,
+      "step": 600
+    },
+    {
+      "epoch": 0.9860577894524146,
+      "grad_norm": 2.044271469116211,
+      "learning_rate": 3.7794779726790664e-06,
+      "logits/chosen": -21.353389739990234,
+      "logits/rejected": -21.419639587402344,
+      "logps/chosen": -1.222917914390564,
+      "logps/rejected": -1.3896757364273071,
+      "loss": 1.2907,
+      "odds_ratio_loss": 0.6782225370407104,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12229180335998535,
+      "rewards/margins": 0.016675781458616257,
+      "rewards/rejected": -0.1389675885438919,
+      "sft_loss": 1.222917914390564,
+      "step": 610
+    },
+    {
+      "epoch": 1.0022226712467166,
+      "grad_norm": 4.229960918426514,
+      "learning_rate": 3.7429001248146096e-06,
+      "logits/chosen": -21.391468048095703,
+      "logits/rejected": -21.433670043945312,
+      "logps/chosen": -1.3207805156707764,
+      "logps/rejected": -1.5086777210235596,
+      "loss": 1.3864,
+      "odds_ratio_loss": 0.655738115310669,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.13207808136940002,
+      "rewards/margins": 0.018789704889059067,
+      "rewards/rejected": -0.1508677750825882,
+      "sft_loss": 1.3207805156707764,
+      "step": 620
+    },
+    {
+      "epoch": 1.0183875530410185,
+      "grad_norm": 1.3560584783554077,
+      "learning_rate": 3.7059653716681227e-06,
+      "logits/chosen": -21.297557830810547,
+      "logits/rejected": -21.24813461303711,
+      "logps/chosen": -1.3175418376922607,
+      "logps/rejected": -1.4784363508224487,
+      "loss": 1.3899,
+      "odds_ratio_loss": 0.7236071825027466,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13175417482852936,
+      "rewards/margins": 0.016089454293251038,
+      "rewards/rejected": -0.1478436291217804,
+      "sft_loss": 1.3175418376922607,
+      "step": 630
+    },
+    {
+      "epoch": 1.0345524348353203,
+      "grad_norm": 4.676194190979004,
+      "learning_rate": 3.668684319247463e-06,
+      "logits/chosen": -21.50934410095215,
+      "logits/rejected": -21.4769229888916,
+      "logps/chosen": -1.2353088855743408,
+      "logps/rejected": -1.5056722164154053,
+      "loss": 1.298,
+      "odds_ratio_loss": 0.6273224949836731,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.12353090196847916,
+      "rewards/margins": 0.02703634463250637,
+      "rewards/rejected": -0.15056723356246948,
+      "sft_loss": 1.2353088855743408,
+      "step": 640
+    },
+    {
+      "epoch": 1.0507173166296222,
+      "grad_norm": 1.456129550933838,
+      "learning_rate": 3.6310676730021373e-06,
+      "logits/chosen": -21.462495803833008,
+      "logits/rejected": -21.57852554321289,
+      "logps/chosen": -1.2601442337036133,
+      "logps/rejected": -1.3218932151794434,
+      "loss": 1.3324,
+      "odds_ratio_loss": 0.7224392890930176,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12601442635059357,
+      "rewards/margins": 0.006174898706376553,
+      "rewards/rejected": -0.1321893036365509,
+      "sft_loss": 1.2601442337036133,
+      "step": 650
+    },
+    {
+      "epoch": 1.066882198423924,
+      "grad_norm": 3.662935733795166,
+      "learning_rate": 3.593126234749178e-06,
+      "logits/chosen": -21.538293838500977,
+      "logits/rejected": -21.53152847290039,
+      "logps/chosen": -1.3035653829574585,
+      "logps/rejected": -1.4283910989761353,
+      "loss": 1.375,
+      "odds_ratio_loss": 0.7146205902099609,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.1303565502166748,
+      "rewards/margins": 0.012482564896345139,
+      "rewards/rejected": -0.14283910393714905,
+      "sft_loss": 1.3035653829574585,
+      "step": 660
+    },
+    {
+      "epoch": 1.083047080218226,
+      "grad_norm": 2.5778450965881348,
+      "learning_rate": 3.554870899571343e-06,
+      "logits/chosen": -21.25736427307129,
+      "logits/rejected": -21.203954696655273,
+      "logps/chosen": -1.2814052104949951,
+      "logps/rejected": -1.4217122793197632,
+      "loss": 1.3498,
+      "odds_ratio_loss": 0.6842816472053528,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.12814053893089294,
+      "rewards/margins": 0.0140307005494833,
+      "rewards/rejected": -0.1421712338924408,
+      "sft_loss": 1.2814052104949951,
+      "step": 670
+    },
+    {
+      "epoch": 1.0992119620125278,
+      "grad_norm": 3.7894856929779053,
+      "learning_rate": 3.5163126526885373e-06,
+      "logits/chosen": -21.061431884765625,
+      "logits/rejected": -21.307607650756836,
+      "logps/chosen": -1.2376093864440918,
+      "logps/rejected": -1.3728429079055786,
+      "loss": 1.3095,
+      "odds_ratio_loss": 0.7185710668563843,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12376093864440918,
+      "rewards/margins": 0.013523365370929241,
+      "rewards/rejected": -0.1372843086719513,
+      "sft_loss": 1.2376093864440918,
+      "step": 680
+    },
+    {
+      "epoch": 1.1153768438068297,
+      "grad_norm": 2.3537509441375732,
+      "learning_rate": 3.4774625663033484e-06,
+      "logits/chosen": -21.21095085144043,
+      "logits/rejected": -21.2464542388916,
+      "logps/chosen": -1.232604742050171,
+      "logps/rejected": -1.3756181001663208,
+      "loss": 1.3002,
+      "odds_ratio_loss": 0.6756417155265808,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.1232604831457138,
+      "rewards/margins": 0.014301331713795662,
+      "rewards/rejected": -0.1375618278980255,
+      "sft_loss": 1.232604742050171,
+      "step": 690
+    },
+    {
+      "epoch": 1.1315417256011315,
+      "grad_norm": 1.4719160795211792,
+      "learning_rate": 3.4383317964216067e-06,
+      "logits/chosen": -21.036582946777344,
+      "logits/rejected": -21.25110626220703,
+      "logps/chosen": -1.2365304231643677,
+      "logps/rejected": -1.3071203231811523,
+      "loss": 1.3116,
+      "odds_ratio_loss": 0.7511636018753052,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.12365303188562393,
+      "rewards/margins": 0.007059005089104176,
+      "rewards/rejected": -0.13071204721927643,
+      "sft_loss": 1.2365304231643677,
+      "step": 700
+    },
+    {
+      "epoch": 1.1477066073954334,
+      "grad_norm": 2.8253660202026367,
+      "learning_rate": 3.398931579648877e-06,
+      "logits/chosen": -21.2774658203125,
+      "logits/rejected": -21.474933624267578,
+      "logps/chosen": -1.3233165740966797,
+      "logps/rejected": -1.49609375,
+      "loss": 1.3932,
+      "odds_ratio_loss": 0.6990328431129456,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13233166933059692,
+      "rewards/margins": 0.017277712002396584,
+      "rewards/rejected": -0.14960938692092896,
+      "sft_loss": 1.3233165740966797,
+      "step": 710
+    },
+    {
+      "epoch": 1.1638714891897353,
+      "grad_norm": 2.0079872608184814,
+      "learning_rate": 3.359273229963813e-06,
+      "logits/chosen": -21.300012588500977,
+      "logits/rejected": -21.20311737060547,
+      "logps/chosen": -1.2444874048233032,
+      "logps/rejected": -1.3516159057617188,
+      "loss": 1.315,
+      "odds_ratio_loss": 0.7051676511764526,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1244487538933754,
+      "rewards/margins": 0.010712863877415657,
+      "rewards/rejected": -0.1351616084575653,
+      "sft_loss": 1.2444874048233032,
+      "step": 720
+    },
+    {
+      "epoch": 1.1800363709840371,
+      "grad_norm": 1.6793773174285889,
+      "learning_rate": 3.319368135469285e-06,
+      "logits/chosen": -21.23295021057129,
+      "logits/rejected": -21.303585052490234,
+      "logps/chosen": -1.294540524482727,
+      "logps/rejected": -1.422060251235962,
+      "loss": 1.3682,
+      "odds_ratio_loss": 0.7366825938224792,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12945404648780823,
+      "rewards/margins": 0.012751981616020203,
+      "rewards/rejected": -0.14220602810382843,
+      "sft_loss": 1.294540524482727,
+      "step": 730
+    },
+    {
+      "epoch": 1.196201252778339,
+      "grad_norm": 2.9670250415802,
+      "learning_rate": 3.279227755122228e-06,
+      "logits/chosen": -21.109752655029297,
+      "logits/rejected": -21.158111572265625,
+      "logps/chosen": -1.2494882345199585,
+      "logps/rejected": -1.5186522006988525,
+      "loss": 1.3163,
+      "odds_ratio_loss": 0.6685449481010437,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12494882196187973,
+      "rewards/margins": 0.02691638469696045,
+      "rewards/rejected": -0.15186521410942078,
+      "sft_loss": 1.2494882345199585,
+      "step": 740
+    },
+    {
+      "epoch": 1.2123661345726409,
+      "grad_norm": 2.090475082397461,
+      "learning_rate": 3.2388636154431417e-06,
+      "logits/chosen": -21.39287757873535,
+      "logits/rejected": -21.400285720825195,
+      "logps/chosen": -1.3509362936019897,
+      "logps/rejected": -1.5079971551895142,
+      "loss": 1.4216,
+      "odds_ratio_loss": 0.7062050104141235,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13509364426136017,
+      "rewards/margins": 0.015706082805991173,
+      "rewards/rejected": -0.1507997065782547,
+      "sft_loss": 1.3509362936019897,
+      "step": 750
+    },
+    {
+      "epoch": 1.2285310163669427,
+      "grad_norm": 1.8322948217391968,
+      "learning_rate": 3.198287307206192e-06,
+      "logits/chosen": -20.998327255249023,
+      "logits/rejected": -21.087013244628906,
+      "logps/chosen": -1.254172921180725,
+      "logps/rejected": -1.3801274299621582,
+      "loss": 1.3241,
+      "odds_ratio_loss": 0.6988333463668823,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.1254173070192337,
+      "rewards/margins": 0.012595455162227154,
+      "rewards/rejected": -0.13801275193691254,
+      "sft_loss": 1.254172921180725,
+      "step": 760
+    },
+    {
+      "epoch": 1.2446958981612446,
+      "grad_norm": 1.5089768171310425,
+      "learning_rate": 3.157510482110856e-06,
+      "logits/chosen": -21.451335906982422,
+      "logits/rejected": -21.22077751159668,
+      "logps/chosen": -1.2650538682937622,
+      "logps/rejected": -1.3639296293258667,
+      "loss": 1.337,
+      "odds_ratio_loss": 0.7199094891548157,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12650538980960846,
+      "rewards/margins": 0.009887583553791046,
+      "rewards/rejected": -0.1363929808139801,
+      "sft_loss": 1.2650538682937622,
+      "step": 770
+    },
+    {
+      "epoch": 1.2608607799555465,
+      "grad_norm": 1.1987851858139038,
+      "learning_rate": 3.116544849436077e-06,
+      "logits/chosen": -21.068490982055664,
+      "logits/rejected": -20.816608428955078,
+      "logps/chosen": -1.3564980030059814,
+      "logps/rejected": -1.5898200273513794,
+      "loss": 1.4243,
+      "odds_ratio_loss": 0.6784577369689941,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13564978539943695,
+      "rewards/margins": 0.023332208395004272,
+      "rewards/rejected": -0.15898200869560242,
+      "sft_loss": 1.3564980030059814,
+      "step": 780
+    },
+    {
+      "epoch": 1.2770256617498483,
+      "grad_norm": 1.9816875457763672,
+      "learning_rate": 3.0754021726778848e-06,
+      "logits/chosen": -21.14816665649414,
+      "logits/rejected": -21.171310424804688,
+      "logps/chosen": -1.1857916116714478,
+      "logps/rejected": -1.4323453903198242,
+      "loss": 1.2508,
+      "odds_ratio_loss": 0.6496400833129883,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.11857914924621582,
+      "rewards/margins": 0.02465539053082466,
+      "rewards/rejected": -0.14323453605175018,
+      "sft_loss": 1.1857916116714478,
+      "step": 790
+    },
+    {
+      "epoch": 1.2931905435441502,
+      "grad_norm": 0.9774518609046936,
+      "learning_rate": 3.0340942661714463e-06,
+      "logits/chosen": -21.33401870727539,
+      "logits/rejected": -21.100515365600586,
+      "logps/chosen": -1.3348338603973389,
+      "logps/rejected": -1.4366466999053955,
+      "loss": 1.4051,
+      "odds_ratio_loss": 0.7027276754379272,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1334833949804306,
+      "rewards/margins": 0.010181264951825142,
+      "rewards/rejected": -0.1436646580696106,
+      "sft_loss": 1.3348338603973389,
+      "step": 800
+    },
+    {
+      "epoch": 1.3093554253384523,
+      "grad_norm": 2.8201749324798584,
+      "learning_rate": 2.992632991698512e-06,
+      "logits/chosen": -20.885223388671875,
+      "logits/rejected": -21.0069522857666,
+      "logps/chosen": -1.28475022315979,
+      "logps/rejected": -1.434466004371643,
+      "loss": 1.3541,
+      "odds_ratio_loss": 0.6936103105545044,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12847502529621124,
+      "rewards/margins": 0.014971581287682056,
+      "rewards/rejected": -0.14344659447669983,
+      "sft_loss": 1.28475022315979,
+      "step": 810
+    },
+    {
+      "epoch": 1.3255203071327541,
+      "grad_norm": 1.557600498199463,
+      "learning_rate": 2.9510302550812537e-06,
+      "logits/chosen": -21.287960052490234,
+      "logits/rejected": -21.395370483398438,
+      "logps/chosen": -1.2278238534927368,
+      "logps/rejected": -1.4308351278305054,
+      "loss": 1.2925,
+      "odds_ratio_loss": 0.646266520023346,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1227823942899704,
+      "rewards/margins": 0.02030111663043499,
+      "rewards/rejected": -0.14308352768421173,
+      "sft_loss": 1.2278238534927368,
+      "step": 820
+    },
+    {
+      "epoch": 1.341685188927056,
+      "grad_norm": 2.6895642280578613,
+      "learning_rate": 2.9092980027634325e-06,
+      "logits/chosen": -21.14548110961914,
+      "logits/rejected": -21.261184692382812,
+      "logps/chosen": -1.1719629764556885,
+      "logps/rejected": -1.3467178344726562,
+      "loss": 1.2381,
+      "odds_ratio_loss": 0.6613830327987671,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11719629913568497,
+      "rewards/margins": 0.017475496977567673,
+      "rewards/rejected": -0.13467177748680115,
+      "sft_loss": 1.1719629764556885,
+      "step": 830
+    },
+    {
+      "epoch": 1.3578500707213579,
+      "grad_norm": 1.8690439462661743,
+      "learning_rate": 2.867448218379927e-06,
+      "logits/chosen": -21.13509750366211,
+      "logits/rejected": -21.14713478088379,
+      "logps/chosen": -1.2881317138671875,
+      "logps/rejected": -1.419306993484497,
+      "loss": 1.359,
+      "odds_ratio_loss": 0.7090221643447876,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12881317734718323,
+      "rewards/margins": 0.013117517344653606,
+      "rewards/rejected": -0.1419306844472885,
+      "sft_loss": 1.2881317138671875,
+      "step": 840
+    },
+    {
+      "epoch": 1.3740149525156597,
+      "grad_norm": 4.924416542053223,
+      "learning_rate": 2.825492919315559e-06,
+      "logits/chosen": -20.981931686401367,
+      "logits/rejected": -21.142297744750977,
+      "logps/chosen": -1.3466829061508179,
+      "logps/rejected": -1.3898065090179443,
+      "loss": 1.4234,
+      "odds_ratio_loss": 0.7671664953231812,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.1346682757139206,
+      "rewards/margins": 0.004312378820031881,
+      "rewards/rejected": -0.1389806568622589,
+      "sft_loss": 1.3466829061508179,
+      "step": 850
+    },
+    {
+      "epoch": 1.3901798343099616,
+      "grad_norm": 1.9793908596038818,
+      "learning_rate": 2.7834441532542482e-06,
+      "logits/chosen": -21.344064712524414,
+      "logits/rejected": -21.355037689208984,
+      "logps/chosen": -1.2326418161392212,
+      "logps/rejected": -1.3959559202194214,
+      "loss": 1.3028,
+      "odds_ratio_loss": 0.7015843987464905,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12326417118310928,
+      "rewards/margins": 0.016331423074007034,
+      "rewards/rejected": -0.13959559798240662,
+      "sft_loss": 1.2326418161392212,
+      "step": 860
+    },
+    {
+      "epoch": 1.4063447161042635,
+      "grad_norm": 1.5754936933517456,
+      "learning_rate": 2.74131399471945e-06,
+      "logits/chosen": -21.10004234313965,
+      "logits/rejected": -21.218639373779297,
+      "logps/chosen": -1.2864024639129639,
+      "logps/rejected": -1.3692989349365234,
+      "loss": 1.3586,
+      "odds_ratio_loss": 0.7215597033500671,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.12864024937152863,
+      "rewards/margins": 0.008289644494652748,
+      "rewards/rejected": -0.13692989945411682,
+      "sft_loss": 1.2864024639129639,
+      "step": 870
+    },
+    {
+      "epoch": 1.4225095978985653,
+      "grad_norm": 2.591798782348633,
+      "learning_rate": 2.6991145416068947e-06,
+      "logits/chosen": -20.910709381103516,
+      "logits/rejected": -21.049091339111328,
+      "logps/chosen": -1.2924576997756958,
+      "logps/rejected": -1.3584909439086914,
+      "loss": 1.3658,
+      "odds_ratio_loss": 0.7329753637313843,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12924575805664062,
+      "rewards/margins": 0.006603321526199579,
+      "rewards/rejected": -0.13584908843040466,
+      "sft_loss": 1.2924576997756958,
+      "step": 880
+    },
+    {
+      "epoch": 1.4386744796928672,
+      "grad_norm": 1.6928390264511108,
+      "learning_rate": 2.6568579117106143e-06,
+      "logits/chosen": -20.872098922729492,
+      "logits/rejected": -20.832103729248047,
+      "logps/chosen": -1.2083466053009033,
+      "logps/rejected": -1.406185507774353,
+      "loss": 1.2749,
+      "odds_ratio_loss": 0.6651225090026855,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12083464860916138,
+      "rewards/margins": 0.019783901050686836,
+      "rewards/rejected": -0.14061854779720306,
+      "sft_loss": 1.2083466053009033,
+      "step": 890
+    },
+    {
+      "epoch": 1.454839361487169,
+      "grad_norm": 0.9949257373809814,
+      "learning_rate": 2.6145562392432544e-06,
+      "logits/chosen": -20.952428817749023,
+      "logits/rejected": -20.857725143432617,
+      "logps/chosen": -1.2628931999206543,
+      "logps/rejected": -1.3288319110870361,
+      "loss": 1.3382,
+      "odds_ratio_loss": 0.75352543592453,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.12628932297229767,
+      "rewards/margins": 0.006593865808099508,
+      "rewards/rejected": -0.1328831911087036,
+      "sft_loss": 1.2628931999206543,
+      "step": 900
+    },
+    {
+      "epoch": 1.471004243281471,
+      "grad_norm": 2.0959157943725586,
+      "learning_rate": 2.5722216713516682e-06,
+      "logits/chosen": -20.460887908935547,
+      "logits/rejected": -20.67595100402832,
+      "logps/chosen": -1.2154316902160645,
+      "logps/rejected": -1.3711063861846924,
+      "loss": 1.284,
+      "odds_ratio_loss": 0.6853248476982117,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12154315412044525,
+      "rewards/margins": 0.015567478723824024,
+      "rewards/rejected": -0.1371106207370758,
+      "sft_loss": 1.2154316902160645,
+      "step": 910
+    },
+    {
+      "epoch": 1.4871691250757728,
+      "grad_norm": 2.9631924629211426,
+      "learning_rate": 2.5298663646288064e-06,
+      "logits/chosen": -20.95013999938965,
+      "logits/rejected": -21.05425453186035,
+      "logps/chosen": -1.213805079460144,
+      "logps/rejected": -1.382340669631958,
+      "loss": 1.2822,
+      "odds_ratio_loss": 0.6836367845535278,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1213805228471756,
+      "rewards/margins": 0.01685354672372341,
+      "rewards/rejected": -0.13823407888412476,
+      "sft_loss": 1.213805079460144,
+      "step": 920
+    },
+    {
+      "epoch": 1.503334006870075,
+      "grad_norm": 2.637305498123169,
+      "learning_rate": 2.487502481622879e-06,
+      "logits/chosen": -20.81560516357422,
+      "logits/rejected": -20.701404571533203,
+      "logps/chosen": -1.3175880908966064,
+      "logps/rejected": -1.4211034774780273,
+      "loss": 1.3892,
+      "odds_ratio_loss": 0.7159314155578613,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13175880908966064,
+      "rewards/margins": 0.010351541452109814,
+      "rewards/rejected": -0.14211036264896393,
+      "sft_loss": 1.3175880908966064,
+      "step": 930
+    },
+    {
+      "epoch": 1.5194988886643768,
+      "grad_norm": 1.9374908208847046,
+      "learning_rate": 2.4451421873448253e-06,
+      "logits/chosen": -20.670528411865234,
+      "logits/rejected": -20.75124168395996,
+      "logps/chosen": -1.2572494745254517,
+      "logps/rejected": -1.367071509361267,
+      "loss": 1.3282,
+      "odds_ratio_loss": 0.7093643546104431,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.12572495639324188,
+      "rewards/margins": 0.010982205159962177,
+      "rewards/rejected": -0.13670717179775238,
+      "sft_loss": 1.2572494745254517,
+      "step": 940
+    },
+    {
+      "epoch": 1.5356637704586786,
+      "grad_norm": 3.949268102645874,
+      "learning_rate": 2.40279764577506e-06,
+      "logits/chosen": -21.262102127075195,
+      "logits/rejected": -21.309324264526367,
+      "logps/chosen": -1.3575810194015503,
+      "logps/rejected": -1.4113634824752808,
+      "loss": 1.4328,
+      "odds_ratio_loss": 0.7524511218070984,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.13575811684131622,
+      "rewards/margins": 0.005378241650760174,
+      "rewards/rejected": -0.14113636314868927,
+      "sft_loss": 1.3575810194015503,
+      "step": 950
+    },
+    {
+      "epoch": 1.5518286522529805,
+      "grad_norm": 1.4890868663787842,
+      "learning_rate": 2.3604810163705242e-06,
+      "logits/chosen": -20.816661834716797,
+      "logits/rejected": -20.944917678833008,
+      "logps/chosen": -1.2137186527252197,
+      "logps/rejected": -1.348968505859375,
+      "loss": 1.2808,
+      "odds_ratio_loss": 0.6708163022994995,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.12137186527252197,
+      "rewards/margins": 0.013524973765015602,
+      "rewards/rejected": -0.13489684462547302,
+      "sft_loss": 1.2137186527252197,
+      "step": 960
+    },
+    {
+      "epoch": 1.5679935340472824,
+      "grad_norm": 2.5744829177856445,
+      "learning_rate": 2.3182044505730364e-06,
+      "logits/chosen": -20.806903839111328,
+      "logits/rejected": -20.799480438232422,
+      "logps/chosen": -1.170987606048584,
+      "logps/rejected": -1.3193377256393433,
+      "loss": 1.2392,
+      "odds_ratio_loss": 0.6825143098831177,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.11709876358509064,
+      "rewards/margins": 0.014834997244179249,
+      "rewards/rejected": -0.1319337785243988,
+      "sft_loss": 1.170987606048584,
+      "step": 970
+    },
+    {
+      "epoch": 1.5841584158415842,
+      "grad_norm": 3.971120834350586,
+      "learning_rate": 2.275980088319941e-06,
+      "logits/chosen": -21.002077102661133,
+      "logits/rejected": -20.75215721130371,
+      "logps/chosen": -1.2036049365997314,
+      "logps/rejected": -1.2877798080444336,
+      "loss": 1.2758,
+      "odds_ratio_loss": 0.7220691442489624,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.12036049365997314,
+      "rewards/margins": 0.008417483419179916,
+      "rewards/rejected": -0.12877798080444336,
+      "sft_loss": 1.2036049365997314,
+      "step": 980
+    },
+    {
+      "epoch": 1.600323297635886,
+      "grad_norm": 2.0432345867156982,
+      "learning_rate": 2.2338200545580577e-06,
+      "logits/chosen": -20.597959518432617,
+      "logits/rejected": -20.82900619506836,
+      "logps/chosen": -1.182715654373169,
+      "logps/rejected": -1.4004007577896118,
+      "loss": 1.2503,
+      "odds_ratio_loss": 0.6758050322532654,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.11827157437801361,
+      "rewards/margins": 0.021768497303128242,
+      "rewards/rejected": -0.1400400698184967,
+      "sft_loss": 1.182715654373169,
+      "step": 990
+    },
+    {
+      "epoch": 1.616488179430188,
+      "grad_norm": 2.0639455318450928,
+      "learning_rate": 2.191736455761947e-06,
+      "logits/chosen": -20.99373435974121,
+      "logits/rejected": -20.91073989868164,
+      "logps/chosen": -1.153194785118103,
+      "logps/rejected": -1.2575992345809937,
+      "loss": 1.2196,
+      "odds_ratio_loss": 0.663902223110199,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.11531949043273926,
+      "rewards/margins": 0.010440433397889137,
+      "rewards/rejected": -0.12575991451740265,
+      "sft_loss": 1.153194785118103,
+      "step": 1000
+    },
+    {
+      "epoch": 1.616488179430188,
+      "eval_logits/chosen": -20.882339477539062,
+      "eval_logits/rejected": -20.911054611206055,
+      "eval_logps/chosen": -1.2678194046020508,
+      "eval_logps/rejected": -1.4078764915466309,
+      "eval_loss": 1.3396929502487183,
+      "eval_odds_ratio_loss": 0.7187345623970032,
+      "eval_rewards/accuracies": 0.5354545712471008,
+      "eval_rewards/chosen": -0.12678195536136627,
+      "eval_rewards/margins": 0.01400570198893547,
+      "eval_rewards/rejected": -0.14078764617443085,
+      "eval_runtime": 158.823,
+      "eval_samples_per_second": 6.926,
+      "eval_sft_loss": 1.2678194046020508,
+      "eval_steps_per_second": 3.463,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6326530612244898,
+      "grad_norm": 1.447840929031372,
+      "learning_rate": 2.1497413764574673e-06,
+      "logits/chosen": -21.098125457763672,
+      "logits/rejected": -21.02555274963379,
+      "logps/chosen": -1.2790714502334595,
+      "logps/rejected": -1.4720830917358398,
+      "loss": 1.3456,
+      "odds_ratio_loss": 0.6656414270401001,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1279071420431137,
+      "rewards/margins": 0.01930117979645729,
+      "rewards/rejected": -0.1472083032131195,
+      "sft_loss": 1.2790714502334595,
+      "step": 1010
+    },
+    {
+      "epoch": 1.6488179430187917,
+      "grad_norm": 2.2929062843322754,
+      "learning_rate": 2.1078468757516395e-06,
+      "logits/chosen": -21.029102325439453,
+      "logits/rejected": -20.96735382080078,
+      "logps/chosen": -1.1978142261505127,
+      "logps/rejected": -1.2682490348815918,
+      "loss": 1.2676,
+      "odds_ratio_loss": 0.6979144811630249,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.11978141963481903,
+      "rewards/margins": 0.007043488323688507,
+      "rewards/rejected": -0.12682488560676575,
+      "sft_loss": 1.1978142261505127,
+      "step": 1020
+    },
+    {
+      "epoch": 1.6649828248130936,
+      "grad_norm": 1.679506778717041,
+      "learning_rate": 2.0660649838698145e-06,
+      "logits/chosen": -21.126895904541016,
+      "logits/rejected": -21.121044158935547,
+      "logps/chosen": -1.2263692617416382,
+      "logps/rejected": -1.3581560850143433,
+      "loss": 1.2969,
+      "odds_ratio_loss": 0.7051782011985779,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12263691425323486,
+      "rewards/margins": 0.013178691267967224,
+      "rewards/rejected": -0.1358156055212021,
+      "sft_loss": 1.2263692617416382,
+      "step": 1030
+    },
+    {
+      "epoch": 1.6811477066073954,
+      "grad_norm": 1.5033296346664429,
+      "learning_rate": 2.0244076987011284e-06,
+      "logits/chosen": -21.097610473632812,
+      "logits/rejected": -21.023983001708984,
+      "logps/chosen": -1.263479232788086,
+      "logps/rejected": -1.430101990699768,
+      "loss": 1.3286,
+      "odds_ratio_loss": 0.6507799029350281,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12634792923927307,
+      "rewards/margins": 0.01666227914392948,
+      "rewards/rejected": -0.1430101990699768,
+      "sft_loss": 1.263479232788086,
+      "step": 1040
+    },
+    {
+      "epoch": 1.6973125884016973,
+      "grad_norm": 1.782820463180542,
+      "learning_rate": 1.982886982353251e-06,
+      "logits/chosen": -20.983699798583984,
+      "logits/rejected": -20.792827606201172,
+      "logps/chosen": -1.2545843124389648,
+      "logps/rejected": -1.4638617038726807,
+      "loss": 1.3236,
+      "odds_ratio_loss": 0.6896650195121765,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12545843422412872,
+      "rewards/margins": 0.020927732810378075,
+      "rewards/rejected": -0.14638619124889374,
+      "sft_loss": 1.2545843124389648,
+      "step": 1050
+    },
+    {
+      "epoch": 1.7134774701959992,
+      "grad_norm": 2.0116512775421143,
+      "learning_rate": 1.941514757717392e-06,
+      "logits/chosen": -20.63968276977539,
+      "logits/rejected": -20.679515838623047,
+      "logps/chosen": -1.2405683994293213,
+      "logps/rejected": -1.4277904033660889,
+      "loss": 1.3065,
+      "odds_ratio_loss": 0.6590927839279175,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12405683845281601,
+      "rewards/margins": 0.018722202628850937,
+      "rewards/rejected": -0.14277902245521545,
+      "sft_loss": 1.2405683994293213,
+      "step": 1060
+    },
+    {
+      "epoch": 1.729642351990301,
+      "grad_norm": 2.4715521335601807,
+      "learning_rate": 1.9003029050445953e-06,
+      "logits/chosen": -20.77858543395996,
+      "logits/rejected": -20.901081085205078,
+      "logps/chosen": -1.2767174243927002,
+      "logps/rejected": -1.3766672611236572,
+      "loss": 1.3478,
+      "odds_ratio_loss": 0.7110922336578369,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1276717483997345,
+      "rewards/margins": 0.00999497715383768,
+      "rewards/rejected": -0.137666717171669,
+      "sft_loss": 1.2767174243927002,
+      "step": 1070
+    },
+    {
+      "epoch": 1.745807233784603,
+      "grad_norm": 1.7757654190063477,
+      "learning_rate": 1.8592632585342523e-06,
+      "logits/chosen": -20.718509674072266,
+      "logits/rejected": -20.882915496826172,
+      "logps/chosen": -1.2459039688110352,
+      "logps/rejected": -1.3744847774505615,
+      "loss": 1.3167,
+      "odds_ratio_loss": 0.7076380848884583,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12459039688110352,
+      "rewards/margins": 0.012858082540333271,
+      "rewards/rejected": -0.13744845986366272,
+      "sft_loss": 1.2459039688110352,
+      "step": 1080
+    },
+    {
+      "epoch": 1.7619721155789048,
+      "grad_norm": 8.900938034057617,
+      "learning_rate": 1.8184076029358527e-06,
+      "logits/chosen": -20.89524269104004,
+      "logits/rejected": -20.46439552307129,
+      "logps/chosen": -1.1931703090667725,
+      "logps/rejected": -1.2244675159454346,
+      "loss": 1.2657,
+      "odds_ratio_loss": 0.725264310836792,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11931703239679337,
+      "rewards/margins": 0.003129716496914625,
+      "rewards/rejected": -0.12244673818349838,
+      "sft_loss": 1.1931703090667725,
+      "step": 1090
+    },
+    {
+      "epoch": 1.7781369973732066,
+      "grad_norm": 1.4426664113998413,
+      "learning_rate": 1.7777476701649318e-06,
+      "logits/chosen": -20.599040985107422,
+      "logits/rejected": -20.622554779052734,
+      "logps/chosen": -1.2419286966323853,
+      "logps/rejected": -1.3642092943191528,
+      "loss": 1.3125,
+      "odds_ratio_loss": 0.7057370543479919,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12419287115335464,
+      "rewards/margins": 0.012228056788444519,
+      "rewards/rejected": -0.13642092049121857,
+      "sft_loss": 1.2419286966323853,
+      "step": 1100
+    },
+    {
+      "epoch": 1.7943018791675085,
+      "grad_norm": 2.3391785621643066,
+      "learning_rate": 1.7372951359341925e-06,
+      "logits/chosen": -20.814512252807617,
+      "logits/rejected": -20.616971969604492,
+      "logps/chosen": -1.183668851852417,
+      "logps/rejected": -1.3042553663253784,
+      "loss": 1.2526,
+      "odds_ratio_loss": 0.6895106434822083,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.11836691200733185,
+      "rewards/margins": 0.012058648280799389,
+      "rewards/rejected": -0.13042554259300232,
+      "sft_loss": 1.183668851852417,
+      "step": 1110
+    },
+    {
+      "epoch": 1.8104667609618104,
+      "grad_norm": 3.773312568664551,
+      "learning_rate": 1.6970616164007547e-06,
+      "logits/chosen": -20.64287567138672,
+      "logits/rejected": -20.485708236694336,
+      "logps/chosen": -1.1766704320907593,
+      "logps/rejected": -1.3071858882904053,
+      "loss": 1.2473,
+      "odds_ratio_loss": 0.7061667442321777,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11766704171895981,
+      "rewards/margins": 0.013051542453467846,
+      "rewards/rejected": -0.13071858882904053,
+      "sft_loss": 1.1766704320907593,
+      "step": 1120
+    },
+    {
+      "epoch": 1.8266316427561122,
+      "grad_norm": 6.383511543273926,
+      "learning_rate": 1.6570586648305276e-06,
+      "logits/chosen": -20.542699813842773,
+      "logits/rejected": -20.710830688476562,
+      "logps/chosen": -1.2573511600494385,
+      "logps/rejected": -1.453174114227295,
+      "loss": 1.3268,
+      "odds_ratio_loss": 0.6940013766288757,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1257351189851761,
+      "rewards/margins": 0.019582299515604973,
+      "rewards/rejected": -0.1453174203634262,
+      "sft_loss": 1.2573511600494385,
+      "step": 1130
+    },
+    {
+      "epoch": 1.842796524550414,
+      "grad_norm": 2.227006196975708,
+      "learning_rate": 1.6172977682806151e-06,
+      "logits/chosen": -20.67087173461914,
+      "logits/rejected": -20.963016510009766,
+      "logps/chosen": -1.254607915878296,
+      "logps/rejected": -1.4263418912887573,
+      "loss": 1.32,
+      "odds_ratio_loss": 0.6536819338798523,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12546080350875854,
+      "rewards/margins": 0.017173394560813904,
+      "rewards/rejected": -0.14263419806957245,
+      "sft_loss": 1.254607915878296,
+      "step": 1140
+    },
+    {
+      "epoch": 1.858961406344716,
+      "grad_norm": 2.0494322776794434,
+      "learning_rate": 1.5777903443007586e-06,
+      "logits/chosen": -20.967281341552734,
+      "logits/rejected": -20.45404624938965,
+      "logps/chosen": -1.291761040687561,
+      "logps/rejected": -1.4253360033035278,
+      "loss": 1.3634,
+      "odds_ratio_loss": 0.7167172431945801,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.12917611002922058,
+      "rewards/margins": 0.013357502408325672,
+      "rewards/rejected": -0.14253361523151398,
+      "sft_loss": 1.291761040687561,
+      "step": 1150
+    },
+    {
+      "epoch": 1.8751262881390178,
+      "grad_norm": 2.5553345680236816,
+      "learning_rate": 1.5385477376547226e-06,
+      "logits/chosen": -20.862293243408203,
+      "logits/rejected": -20.802623748779297,
+      "logps/chosen": -1.3029327392578125,
+      "logps/rejected": -1.4472944736480713,
+      "loss": 1.3688,
+      "odds_ratio_loss": 0.6585663557052612,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13029327988624573,
+      "rewards/margins": 0.014436167664825916,
+      "rewards/rejected": -0.14472945034503937,
+      "sft_loss": 1.3029327392578125,
+      "step": 1160
+    },
+    {
+      "epoch": 1.89129116993332,
+      "grad_norm": 1.7979766130447388,
+      "learning_rate": 1.4995812170625845e-06,
+      "logits/chosen": -20.667551040649414,
+      "logits/rejected": -20.655712127685547,
+      "logps/chosen": -1.317637324333191,
+      "logps/rejected": -1.5292212963104248,
+      "loss": 1.3849,
+      "odds_ratio_loss": 0.6725908517837524,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.1317637413740158,
+      "rewards/margins": 0.021158380433917046,
+      "rewards/rejected": -0.152922123670578,
+      "sft_loss": 1.317637324333191,
+      "step": 1170
+    },
+    {
+      "epoch": 1.9074560517276218,
+      "grad_norm": 7.151007652282715,
+      "learning_rate": 1.4609019719648666e-06,
+      "logits/chosen": -21.043447494506836,
+      "logits/rejected": -20.968263626098633,
+      "logps/chosen": -1.2828052043914795,
+      "logps/rejected": -1.4710747003555298,
+      "loss": 1.3489,
+      "odds_ratio_loss": 0.6608654856681824,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12828053534030914,
+      "rewards/margins": 0.018826942890882492,
+      "rewards/rejected": -0.14710748195648193,
+      "sft_loss": 1.2828052043914795,
+      "step": 1180
+    },
+    {
+      "epoch": 1.9236209335219236,
+      "grad_norm": 3.7433314323425293,
+      "learning_rate": 1.42252110930943e-06,
+      "logits/chosen": -20.565446853637695,
+      "logits/rejected": -20.41248893737793,
+      "logps/chosen": -1.1272156238555908,
+      "logps/rejected": -1.213004469871521,
+      "loss": 1.2001,
+      "odds_ratio_loss": 0.7286427021026611,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.11272156238555908,
+      "rewards/margins": 0.008578887209296227,
+      "rewards/rejected": -0.12130045890808105,
+      "sft_loss": 1.1272156238555908,
+      "step": 1190
+    },
+    {
+      "epoch": 1.9397858153162255,
+      "grad_norm": 2.740086793899536,
+      "learning_rate": 1.3844496503620493e-06,
+      "logits/chosen": -20.73773765563965,
+      "logits/rejected": -20.874481201171875,
+      "logps/chosen": -1.2707687616348267,
+      "logps/rejected": -1.3804893493652344,
+      "loss": 1.3383,
+      "odds_ratio_loss": 0.6750306487083435,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1270768642425537,
+      "rewards/margins": 0.010972070507705212,
+      "rewards/rejected": -0.1380489468574524,
+      "sft_loss": 1.2707687616348267,
+      "step": 1200
+    },
+    {
+      "epoch": 1.9559506971105274,
+      "grad_norm": 3.2234959602355957,
+      "learning_rate": 1.3466985275416081e-06,
+      "logits/chosen": -20.732952117919922,
+      "logits/rejected": -20.362863540649414,
+      "logps/chosen": -1.3259879350662231,
+      "logps/rejected": -1.436372995376587,
+      "loss": 1.4005,
+      "odds_ratio_loss": 0.7446285486221313,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.13259878754615784,
+      "rewards/margins": 0.011038513854146004,
+      "rewards/rejected": -0.14363731443881989,
+      "sft_loss": 1.3259879350662231,
+      "step": 1210
+    },
+    {
+      "epoch": 1.9721155789048292,
+      "grad_norm": 2.2601141929626465,
+      "learning_rate": 1.309278581280791e-06,
+      "logits/chosen": -20.75266456604004,
+      "logits/rejected": -20.531200408935547,
+      "logps/chosen": -1.2139203548431396,
+      "logps/rejected": -1.3907318115234375,
+      "loss": 1.2807,
+      "odds_ratio_loss": 0.6679332852363586,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12139203399419785,
+      "rewards/margins": 0.017681140452623367,
+      "rewards/rejected": -0.1390731781721115,
+      "sft_loss": 1.2139203548431396,
+      "step": 1220
+    },
+    {
+      "epoch": 1.9882804606991311,
+      "grad_norm": 1.351995825767517,
+      "learning_rate": 1.272200556913199e-06,
+      "logits/chosen": -20.665508270263672,
+      "logits/rejected": -20.715679168701172,
+      "logps/chosen": -1.2477220296859741,
+      "logps/rejected": -1.4040950536727905,
+      "loss": 1.3204,
+      "odds_ratio_loss": 0.7266369462013245,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12477221339941025,
+      "rewards/margins": 0.015637310221791267,
+      "rewards/rejected": -0.14040951430797577,
+      "sft_loss": 1.2477220296859741,
+      "step": 1230
+    },
+    {
+      "epoch": 2.004445342493433,
+      "grad_norm": 3.1234052181243896,
+      "learning_rate": 1.2354751015877698e-06,
+      "logits/chosen": -20.652090072631836,
+      "logits/rejected": -20.866174697875977,
+      "logps/chosen": -1.1843925714492798,
+      "logps/rejected": -1.4374680519104004,
+      "loss": 1.2509,
+      "odds_ratio_loss": 0.6650271415710449,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11843927204608917,
+      "rewards/margins": 0.02530754543840885,
+      "rewards/rejected": -0.14374680817127228,
+      "sft_loss": 1.1843925714492798,
+      "step": 1240
+    },
+    {
+      "epoch": 2.020610224287735,
+      "grad_norm": 2.8965847492218018,
+      "learning_rate": 1.1991127612113945e-06,
+      "logits/chosen": -20.925878524780273,
+      "logits/rejected": -21.110370635986328,
+      "logps/chosen": -1.2498103380203247,
+      "logps/rejected": -1.4370229244232178,
+      "loss": 1.3151,
+      "odds_ratio_loss": 0.6532084345817566,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12498103082180023,
+      "rewards/margins": 0.018721258267760277,
+      "rewards/rejected": -0.14370229840278625,
+      "sft_loss": 1.2498103380203247,
+      "step": 1250
+    },
+    {
+      "epoch": 2.036775106082037,
+      "grad_norm": 2.9547386169433594,
+      "learning_rate": 1.1631239774206035e-06,
+      "logits/chosen": -20.5147705078125,
+      "logits/rejected": -20.376428604125977,
+      "logps/chosen": -1.236487627029419,
+      "logps/rejected": -1.4057966470718384,
+      "loss": 1.3074,
+      "odds_ratio_loss": 0.7092265486717224,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12364877760410309,
+      "rewards/margins": 0.01693090796470642,
+      "rewards/rejected": -0.14057967066764832,
+      "sft_loss": 1.236487627029419,
+      "step": 1260
+    },
+    {
+      "epoch": 2.052939987876339,
+      "grad_norm": 3.2073123455047607,
+      "learning_rate": 1.1275190845831978e-06,
+      "logits/chosen": -20.78199005126953,
+      "logits/rejected": -20.76191520690918,
+      "logps/chosen": -1.211749792098999,
+      "logps/rejected": -1.4196996688842773,
+      "loss": 1.2743,
+      "odds_ratio_loss": 0.6256077289581299,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12117497622966766,
+      "rewards/margins": 0.020794976502656937,
+      "rewards/rejected": -0.1419699490070343,
+      "sft_loss": 1.211749792098999,
+      "step": 1270
+    },
+    {
+      "epoch": 2.0691048696706407,
+      "grad_norm": 1.841145634651184,
+      "learning_rate": 1.0923083068306778e-06,
+      "logits/chosen": -20.74366569519043,
+      "logits/rejected": -20.469417572021484,
+      "logps/chosen": -1.1939375400543213,
+      "logps/rejected": -1.4483907222747803,
+      "loss": 1.2578,
+      "odds_ratio_loss": 0.6386052966117859,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11939376592636108,
+      "rewards/margins": 0.025445321574807167,
+      "rewards/rejected": -0.1448390930891037,
+      "sft_loss": 1.1939375400543213,
+      "step": 1280
+    },
+    {
+      "epoch": 2.0852697514649425,
+      "grad_norm": 1.7583280801773071,
+      "learning_rate": 1.0575017551223348e-06,
+      "logits/chosen": -20.574234008789062,
+      "logits/rejected": -20.410676956176758,
+      "logps/chosen": -1.1411545276641846,
+      "logps/rejected": -1.3073662519454956,
+      "loss": 1.2096,
+      "odds_ratio_loss": 0.6839541792869568,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.11411545425653458,
+      "rewards/margins": 0.016621166840195656,
+      "rewards/rejected": -0.13073661923408508,
+      "sft_loss": 1.1411545276641846,
+      "step": 1290
+    },
+    {
+      "epoch": 2.1014346332592444,
+      "grad_norm": 1.8893145322799683,
+      "learning_rate": 1.023109424341833e-06,
+      "logits/chosen": -20.674358367919922,
+      "logits/rejected": -20.841875076293945,
+      "logps/chosen": -1.318074107170105,
+      "logps/rejected": -1.4704669713974,
+      "loss": 1.3891,
+      "odds_ratio_loss": 0.7104289531707764,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13180741667747498,
+      "rewards/margins": 0.01523929089307785,
+      "rewards/rejected": -0.14704671502113342,
+      "sft_loss": 1.318074107170105,
+      "step": 1300
+    },
+    {
+      "epoch": 2.1175995150535463,
+      "grad_norm": 1.844677209854126,
+      "learning_rate": 9.891411904271273e-07,
+      "logits/chosen": -20.51424217224121,
+      "logits/rejected": -20.641857147216797,
+      "logps/chosen": -1.168316125869751,
+      "logps/rejected": -1.3018604516983032,
+      "loss": 1.2364,
+      "odds_ratio_loss": 0.6810965538024902,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11683161556720734,
+      "rewards/margins": 0.013354445807635784,
+      "rewards/rejected": -0.1301860511302948,
+      "sft_loss": 1.168316125869751,
+      "step": 1310
+    },
+    {
+      "epoch": 2.133764396847848,
+      "grad_norm": 1.6055060625076294,
+      "learning_rate": 9.556068075345363e-07,
+      "logits/chosen": -20.898950576782227,
+      "logits/rejected": -20.616153717041016,
+      "logps/chosen": -1.2051537036895752,
+      "logps/rejected": -1.2989599704742432,
+      "loss": 1.2733,
+      "odds_ratio_loss": 0.6817353367805481,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1205153614282608,
+      "rewards/margins": 0.009380649775266647,
+      "rewards/rejected": -0.12989601492881775,
+      "sft_loss": 1.2051537036895752,
+      "step": 1320
+    },
+    {
+      "epoch": 2.14992927864215,
+      "grad_norm": 1.451566219329834,
+      "learning_rate": 9.225159052377838e-07,
+      "logits/chosen": -20.857812881469727,
+      "logits/rejected": -20.955089569091797,
+      "logps/chosen": -1.242044448852539,
+      "logps/rejected": -1.4105781316757202,
+      "loss": 1.311,
+      "odds_ratio_loss": 0.6894499063491821,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.12420444190502167,
+      "rewards/margins": 0.016853367909789085,
+      "rewards/rejected": -0.1410578191280365,
+      "sft_loss": 1.242044448852539,
+      "step": 1330
+    },
+    {
+      "epoch": 2.166094160436452,
+      "grad_norm": 3.742938995361328,
+      "learning_rate": 8.898779857628184e-07,
+      "logits/chosen": -20.516098022460938,
+      "logits/rejected": -20.78036117553711,
+      "logps/chosen": -1.1591819524765015,
+      "logps/rejected": -1.2938919067382812,
+      "loss": 1.2271,
+      "odds_ratio_loss": 0.6796503663063049,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11591820418834686,
+      "rewards/margins": 0.013470989651978016,
+      "rewards/rejected": -0.1293891966342926,
+      "sft_loss": 1.1591819524765015,
+      "step": 1340
+    },
+    {
+      "epoch": 2.1822590422307537,
+      "grad_norm": 1.6822633743286133,
+      "learning_rate": 8.577024212591975e-07,
+      "logits/chosen": -21.10406494140625,
+      "logits/rejected": -20.88094711303711,
+      "logps/chosen": -1.2675734758377075,
+      "logps/rejected": -1.3737549781799316,
+      "loss": 1.3373,
+      "odds_ratio_loss": 0.6970787048339844,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12675733864307404,
+      "rewards/margins": 0.010618140920996666,
+      "rewards/rejected": -0.13737548887729645,
+      "sft_loss": 1.2675734758377075,
+      "step": 1350
+    },
+    {
+      "epoch": 2.1984239240250556,
+      "grad_norm": 2.0817370414733887,
+      "learning_rate": 8.259984511088276e-07,
+      "logits/chosen": -20.762319564819336,
+      "logits/rejected": -20.748157501220703,
+      "logps/chosen": -1.2708746194839478,
+      "logps/rejected": -1.3757803440093994,
+      "loss": 1.3444,
+      "odds_ratio_loss": 0.7355143427848816,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.12708747386932373,
+      "rewards/margins": 0.010490546002984047,
+      "rewards/rejected": -0.13757804036140442,
+      "sft_loss": 1.2708746194839478,
+      "step": 1360
+    },
+    {
+      "epoch": 2.2145888058193575,
+      "grad_norm": 1.3843287229537964,
+      "learning_rate": 7.947751792728237e-07,
+      "logits/chosen": -20.849414825439453,
+      "logits/rejected": -20.718679428100586,
+      "logps/chosen": -1.1914303302764893,
+      "logps/rejected": -1.3572627305984497,
+      "loss": 1.257,
+      "odds_ratio_loss": 0.6554330587387085,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.119143046438694,
+      "rewards/margins": 0.016583234071731567,
+      "rewards/rejected": -0.13572627305984497,
+      "sft_loss": 1.1914303302764893,
+      "step": 1370
+    },
+    {
+      "epoch": 2.2307536876136593,
+      "grad_norm": 10.062128067016602,
+      "learning_rate": 7.640415716772626e-07,
+      "logits/chosen": -20.725990295410156,
+      "logits/rejected": -20.761856079101562,
+      "logps/chosen": -1.3113725185394287,
+      "logps/rejected": -1.4960176944732666,
+      "loss": 1.381,
+      "odds_ratio_loss": 0.6963750123977661,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13113726675510406,
+      "rewards/margins": 0.01846454106271267,
+      "rewards/rejected": -0.14960181713104248,
+      "sft_loss": 1.3113725185394287,
+      "step": 1380
+    },
+    {
+      "epoch": 2.246918569407961,
+      "grad_norm": 2.1556107997894287,
+      "learning_rate": 7.338064536385722e-07,
+      "logits/chosen": -20.7343692779541,
+      "logits/rejected": -20.6729793548584,
+      "logps/chosen": -1.2534866333007812,
+      "logps/rejected": -1.4300428628921509,
+      "loss": 1.3205,
+      "odds_ratio_loss": 0.6699201464653015,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12534865736961365,
+      "rewards/margins": 0.017655618488788605,
+      "rewards/rejected": -0.14300426840782166,
+      "sft_loss": 1.2534866333007812,
+      "step": 1390
+    },
+    {
+      "epoch": 2.263083451202263,
+      "grad_norm": 2.0938661098480225,
+      "learning_rate": 7.040785073292883e-07,
+      "logits/chosen": -20.498939514160156,
+      "logits/rejected": -20.509479522705078,
+      "logps/chosen": -1.3050354719161987,
+      "logps/rejected": -1.4407126903533936,
+      "loss": 1.3786,
+      "odds_ratio_loss": 0.7356770038604736,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1305035650730133,
+      "rewards/margins": 0.013567718677222729,
+      "rewards/rejected": -0.14407126605510712,
+      "sft_loss": 1.3050354719161987,
+      "step": 1400
+    },
+    {
+      "epoch": 2.279248332996565,
+      "grad_norm": 2.8200390338897705,
+      "learning_rate": 6.748662692849297e-07,
+      "logits/chosen": -20.943716049194336,
+      "logits/rejected": -20.97843360900879,
+      "logps/chosen": -1.2242540121078491,
+      "logps/rejected": -1.4535945653915405,
+      "loss": 1.2894,
+      "odds_ratio_loss": 0.6512330770492554,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12242539972066879,
+      "rewards/margins": 0.02293405309319496,
+      "rewards/rejected": -0.14535947144031525,
+      "sft_loss": 1.2242540121078491,
+      "step": 1410
+    },
+    {
+      "epoch": 2.295413214790867,
+      "grad_norm": 3.1267454624176025,
+      "learning_rate": 6.46178127952686e-07,
+      "logits/chosen": -20.665185928344727,
+      "logits/rejected": -20.615243911743164,
+      "logps/chosen": -1.2219477891921997,
+      "logps/rejected": -1.3721575736999512,
+      "loss": 1.2881,
+      "odds_ratio_loss": 0.6616476774215698,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12219478189945221,
+      "rewards/margins": 0.015020984224975109,
+      "rewards/rejected": -0.1372157633304596,
+      "sft_loss": 1.2219477891921997,
+      "step": 1420
+    },
+    {
+      "epoch": 2.3115780965851687,
+      "grad_norm": 2.4645252227783203,
+      "learning_rate": 6.180223212826289e-07,
+      "logits/chosen": -20.52808380126953,
+      "logits/rejected": -20.359468460083008,
+      "logps/chosen": -1.2114827632904053,
+      "logps/rejected": -1.3234493732452393,
+      "loss": 1.2822,
+      "odds_ratio_loss": 0.7067694664001465,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12114828824996948,
+      "rewards/margins": 0.011196643114089966,
+      "rewards/rejected": -0.13234491646289825,
+      "sft_loss": 1.2114827632904053,
+      "step": 1430
+    },
+    {
+      "epoch": 2.3277429783794705,
+      "grad_norm": 1.1290780305862427,
+      "learning_rate": 5.904069343621443e-07,
+      "logits/chosen": -20.84099006652832,
+      "logits/rejected": -20.68367576599121,
+      "logps/chosen": -1.1913143396377563,
+      "logps/rejected": -1.3442729711532593,
+      "loss": 1.2568,
+      "odds_ratio_loss": 0.6552363634109497,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1191314235329628,
+      "rewards/margins": 0.015295890159904957,
+      "rewards/rejected": -0.13442730903625488,
+      "sft_loss": 1.1913143396377563,
+      "step": 1440
+    },
+    {
+      "epoch": 2.3439078601737724,
+      "grad_norm": 4.884983539581299,
+      "learning_rate": 5.633398970942544e-07,
+      "logits/chosen": -20.582796096801758,
+      "logits/rejected": -20.532047271728516,
+      "logps/chosen": -1.2106900215148926,
+      "logps/rejected": -1.3333368301391602,
+      "loss": 1.2794,
+      "odds_ratio_loss": 0.6871799826622009,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.12106901407241821,
+      "rewards/margins": 0.012264668941497803,
+      "rewards/rejected": -0.13333368301391602,
+      "sft_loss": 1.2106900215148926,
+      "step": 1450
+    },
+    {
+      "epoch": 2.3600727419680743,
+      "grad_norm": 2.4621684551239014,
+      "learning_rate": 5.368289819205069e-07,
+      "logits/chosen": -20.69870376586914,
+      "logits/rejected": -20.650863647460938,
+      "logps/chosen": -1.1822137832641602,
+      "logps/rejected": -1.2959617376327515,
+      "loss": 1.2522,
+      "odds_ratio_loss": 0.6997024416923523,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.11822137981653214,
+      "rewards/margins": 0.011374806985259056,
+      "rewards/rejected": -0.12959617376327515,
+      "sft_loss": 1.1822137832641602,
+      "step": 1460
+    },
+    {
+      "epoch": 2.376237623762376,
+      "grad_norm": 2.461702585220337,
+      "learning_rate": 5.108818015890785e-07,
+      "logits/chosen": -20.82188606262207,
+      "logits/rejected": -20.84579086303711,
+      "logps/chosen": -1.279842734336853,
+      "logps/rejected": -1.449312448501587,
+      "loss": 1.348,
+      "odds_ratio_loss": 0.6815730333328247,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.12798430025577545,
+      "rewards/margins": 0.016946956515312195,
+      "rewards/rejected": -0.14493125677108765,
+      "sft_loss": 1.279842734336853,
+      "step": 1470
+    },
+    {
+      "epoch": 2.392402505556678,
+      "grad_norm": 2.5109152793884277,
+      "learning_rate": 4.855058069687291e-07,
+      "logits/chosen": -20.60469627380371,
+      "logits/rejected": -20.50313949584961,
+      "logps/chosen": -1.1870944499969482,
+      "logps/rejected": -1.382088541984558,
+      "loss": 1.254,
+      "odds_ratio_loss": 0.6693797707557678,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.11870943009853363,
+      "rewards/margins": 0.01949942298233509,
+      "rewards/rejected": -0.13820883631706238,
+      "sft_loss": 1.1870944499969482,
+      "step": 1480
+    },
+    {
+      "epoch": 2.40856738735098,
+      "grad_norm": 2.1160638332366943,
+      "learning_rate": 4.607082849092523e-07,
+      "logits/chosen": -20.56966781616211,
+      "logits/rejected": -20.43010902404785,
+      "logps/chosen": -1.324727177619934,
+      "logps/rejected": -1.4259936809539795,
+      "loss": 1.3968,
+      "odds_ratio_loss": 0.7211604118347168,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1324726939201355,
+      "rewards/margins": 0.010126668028533459,
+      "rewards/rejected": -0.14259937405586243,
+      "sft_loss": 1.324727177619934,
+      "step": 1490
+    },
+    {
+      "epoch": 2.4247322691452817,
+      "grad_norm": 4.8063836097717285,
+      "learning_rate": 4.3649635614901405e-07,
+      "logits/chosen": -20.565994262695312,
+      "logits/rejected": -20.88812255859375,
+      "logps/chosen": -1.2172143459320068,
+      "logps/rejected": -1.277565836906433,
+      "loss": 1.2883,
+      "odds_ratio_loss": 0.711229145526886,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12172143161296844,
+      "rewards/margins": 0.006035161670297384,
+      "rewards/rejected": -0.12775658071041107,
+      "sft_loss": 1.2172143459320068,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4247322691452817,
+      "eval_logits/chosen": -20.66438865661621,
+      "eval_logits/rejected": -20.695892333984375,
+      "eval_logps/chosen": -1.2573115825653076,
+      "eval_logps/rejected": -1.3996058702468872,
+      "eval_loss": 1.3293771743774414,
+      "eval_odds_ratio_loss": 0.7206552624702454,
+      "eval_rewards/accuracies": 0.5345454812049866,
+      "eval_rewards/chosen": -0.12573117017745972,
+      "eval_rewards/margins": 0.014229409396648407,
+      "eval_rewards/rejected": -0.13996057212352753,
+      "eval_runtime": 82.3734,
+      "eval_samples_per_second": 13.354,
+      "eval_sft_loss": 1.2573115825653076,
+      "eval_steps_per_second": 6.677,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4408971509395836,
+      "grad_norm": 2.092519998550415,
+      "learning_rate": 4.128769732701973e-07,
+      "logits/chosen": -20.61777114868164,
+      "logits/rejected": -20.48794174194336,
+      "logps/chosen": -1.2248725891113281,
+      "logps/rejected": -1.3724851608276367,
+      "loss": 1.2928,
+      "odds_ratio_loss": 0.6797080039978027,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.12248726189136505,
+      "rewards/margins": 0.014761263504624367,
+      "rewards/rejected": -0.13724853098392487,
+      "sft_loss": 1.2248725891113281,
+      "step": 1510
+    },
+    {
+      "epoch": 2.4570620327338855,
+      "grad_norm": 2.445197582244873,
+      "learning_rate": 3.8985691870233046e-07,
+      "logits/chosen": -20.743959426879883,
+      "logits/rejected": -20.582164764404297,
+      "logps/chosen": -1.2659637928009033,
+      "logps/rejected": -1.4508241415023804,
+      "loss": 1.3339,
+      "odds_ratio_loss": 0.678980827331543,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1265963912010193,
+      "rewards/margins": 0.01848602667450905,
+      "rewards/rejected": -0.14508241415023804,
+      "sft_loss": 1.2659637928009033,
+      "step": 1520
+    },
+    {
+      "epoch": 2.4732269145281873,
+      "grad_norm": 6.312464714050293,
+      "learning_rate": 3.6744280277467904e-07,
+      "logits/chosen": -20.964641571044922,
+      "logits/rejected": -21.01685333251953,
+      "logps/chosen": -1.2571816444396973,
+      "logps/rejected": -1.413922905921936,
+      "loss": 1.3289,
+      "odds_ratio_loss": 0.7169492244720459,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12571817636489868,
+      "rewards/margins": 0.015674125403165817,
+      "rewards/rejected": -0.1413923054933548,
+      "sft_loss": 1.2571816444396973,
+      "step": 1530
+    },
+    {
+      "epoch": 2.489391796322489,
+      "grad_norm": 2.195627450942993,
+      "learning_rate": 3.456410618180503e-07,
+      "logits/chosen": -19.93832778930664,
+      "logits/rejected": -20.330801010131836,
+      "logps/chosen": -1.1545674800872803,
+      "logps/rejected": -1.4308401346206665,
+      "loss": 1.2195,
+      "odds_ratio_loss": 0.649765133857727,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.11545674502849579,
+      "rewards/margins": 0.027627255767583847,
+      "rewards/rejected": -0.14308401942253113,
+      "sft_loss": 1.1545674800872803,
+      "step": 1540
+    },
+    {
+      "epoch": 2.5055566781167915,
+      "grad_norm": 1.8385835886001587,
+      "learning_rate": 3.244579563165753e-07,
+      "logits/chosen": -20.591102600097656,
+      "logits/rejected": -20.71021270751953,
+      "logps/chosen": -1.2204065322875977,
+      "logps/rejected": -1.452178955078125,
+      "loss": 1.2861,
+      "odds_ratio_loss": 0.65655916929245,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12204064428806305,
+      "rewards/margins": 0.02317725121974945,
+      "rewards/rejected": -0.1452178955078125,
+      "sft_loss": 1.2204065322875977,
+      "step": 1550
+    },
+    {
+      "epoch": 2.521721559911093,
+      "grad_norm": 1.5258744955062866,
+      "learning_rate": 3.038995691099697e-07,
+      "logits/chosen": -20.90871238708496,
+      "logits/rejected": -20.73806381225586,
+      "logps/chosen": -1.288413166999817,
+      "logps/rejected": -1.4578988552093506,
+      "loss": 1.36,
+      "odds_ratio_loss": 0.7158418297767639,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1288413256406784,
+      "rewards/margins": 0.016948552802205086,
+      "rewards/rejected": -0.14578989148139954,
+      "sft_loss": 1.288413166999817,
+      "step": 1560
+    },
+    {
+      "epoch": 2.5378864417053952,
+      "grad_norm": 2.4840152263641357,
+      "learning_rate": 2.839718036468192e-07,
+      "logits/chosen": -20.923898696899414,
+      "logits/rejected": -20.989917755126953,
+      "logps/chosen": -1.3150464296340942,
+      "logps/rejected": -1.4090582132339478,
+      "loss": 1.3879,
+      "odds_ratio_loss": 0.7283130288124084,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13150465488433838,
+      "rewards/margins": 0.009401168674230576,
+      "rewards/rejected": -0.14090582728385925,
+      "sft_loss": 1.3150464296340942,
+      "step": 1570
+    },
+    {
+      "epoch": 2.5540513234996967,
+      "grad_norm": 4.045677185058594,
+      "learning_rate": 2.646803822893723e-07,
+      "logits/chosen": -20.986186981201172,
+      "logits/rejected": -20.914697647094727,
+      "logps/chosen": -1.2622355222702026,
+      "logps/rejected": -1.387225866317749,
+      "loss": 1.3346,
+      "odds_ratio_loss": 0.7240074872970581,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.12622354924678802,
+      "rewards/margins": 0.012499034404754639,
+      "rewards/rejected": -0.13872259855270386,
+      "sft_loss": 1.2622355222702026,
+      "step": 1580
+    },
+    {
+      "epoch": 2.570216205293999,
+      "grad_norm": 1.6737409830093384,
+      "learning_rate": 2.460308446703341e-07,
+      "logits/chosen": -20.74363136291504,
+      "logits/rejected": -20.45053482055664,
+      "logps/chosen": -1.2218042612075806,
+      "logps/rejected": -1.310227394104004,
+      "loss": 1.293,
+      "odds_ratio_loss": 0.7117012739181519,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12218043953180313,
+      "rewards/margins": 0.008842318318784237,
+      "rewards/rejected": -0.13102273643016815,
+      "sft_loss": 1.2218042612075806,
+      "step": 1590
+    },
+    {
+      "epoch": 2.5863810870883004,
+      "grad_norm": 2.487335443496704,
+      "learning_rate": 2.2802854610213143e-07,
+      "logits/chosen": -20.622318267822266,
+      "logits/rejected": -20.454730987548828,
+      "logps/chosen": -1.184971570968628,
+      "logps/rejected": -1.4719953536987305,
+      "loss": 1.2481,
+      "odds_ratio_loss": 0.6310954689979553,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.11849715560674667,
+      "rewards/margins": 0.02870238944888115,
+      "rewards/rejected": -0.14719954133033752,
+      "sft_loss": 1.184971570968628,
+      "step": 1600
+    },
+    {
+      "epoch": 2.6025459688826027,
+      "grad_norm": 7.610108375549316,
+      "learning_rate": 2.106786560391072e-07,
+      "logits/chosen": -20.3998966217041,
+      "logits/rejected": -20.51192855834961,
+      "logps/chosen": -1.2781306505203247,
+      "logps/rejected": -1.3927931785583496,
+      "loss": 1.3503,
+      "odds_ratio_loss": 0.7214857339859009,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12781307101249695,
+      "rewards/margins": 0.011466264724731445,
+      "rewards/rejected": -0.139279305934906,
+      "sft_loss": 1.2781306505203247,
+      "step": 1610
+    },
+    {
+      "epoch": 2.6187108506769046,
+      "grad_norm": 2.2551307678222656,
+      "learning_rate": 1.9398615659308255e-07,
+      "logits/chosen": -20.576255798339844,
+      "logits/rejected": -20.769161224365234,
+      "logps/chosen": -1.2140244245529175,
+      "logps/rejected": -1.2931668758392334,
+      "loss": 1.2827,
+      "odds_ratio_loss": 0.6866000294685364,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12140244245529175,
+      "rewards/margins": 0.007914242334663868,
+      "rewards/rejected": -0.12931668758392334,
+      "sft_loss": 1.2140244245529175,
+      "step": 1620
+    },
+    {
+      "epoch": 2.6348757324712064,
+      "grad_norm": 2.147873640060425,
+      "learning_rate": 1.7795584110272184e-07,
+      "logits/chosen": -20.9218807220459,
+      "logits/rejected": -20.841930389404297,
+      "logps/chosen": -1.261558175086975,
+      "logps/rejected": -1.3861316442489624,
+      "loss": 1.3318,
+      "odds_ratio_loss": 0.7022627592086792,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1261558085680008,
+      "rewards/margins": 0.01245735865086317,
+      "rewards/rejected": -0.13861316442489624,
+      "sft_loss": 1.261558175086975,
+      "step": 1630
+    },
+    {
+      "epoch": 2.6510406142655083,
+      "grad_norm": 6.040190696716309,
+      "learning_rate": 1.6259231275709636e-07,
+      "logits/chosen": -20.989246368408203,
+      "logits/rejected": -20.884021759033203,
+      "logps/chosen": -1.2266334295272827,
+      "logps/rejected": -1.2976648807525635,
+      "loss": 1.3008,
+      "odds_ratio_loss": 0.7414904832839966,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12266335636377335,
+      "rewards/margins": 0.007103143725544214,
+      "rewards/rejected": -0.12976649403572083,
+      "sft_loss": 1.2266334295272827,
+      "step": 1640
+    },
+    {
+      "epoch": 2.66720549605981,
+      "grad_norm": 1.8157051801681519,
+      "learning_rate": 1.478999832738548e-07,
+      "logits/chosen": -20.997966766357422,
+      "logits/rejected": -20.761661529541016,
+      "logps/chosen": -1.2066887617111206,
+      "logps/rejected": -1.385434865951538,
+      "loss": 1.276,
+      "odds_ratio_loss": 0.6930679082870483,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12066888809204102,
+      "rewards/margins": 0.017874618992209435,
+      "rewards/rejected": -0.138543501496315,
+      "sft_loss": 1.2066887617111206,
+      "step": 1650
+    },
+    {
+      "epoch": 2.683370377854112,
+      "grad_norm": 2.9221081733703613,
+      "learning_rate": 1.338830716323769e-07,
+      "logits/chosen": -20.6229248046875,
+      "logits/rejected": -20.6382999420166,
+      "logps/chosen": -1.2255455255508423,
+      "logps/rejected": -1.3001086711883545,
+      "loss": 1.2979,
+      "odds_ratio_loss": 0.7236055135726929,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.12255455553531647,
+      "rewards/margins": 0.007456316612660885,
+      "rewards/rejected": -0.13001087307929993,
+      "sft_loss": 1.2255455255508423,
+      "step": 1660
+    },
+    {
+      "epoch": 2.699535259648414,
+      "grad_norm": 3.0684990882873535,
+      "learning_rate": 1.205456028622723e-07,
+      "logits/chosen": -20.87864112854004,
+      "logits/rejected": -20.77214241027832,
+      "logps/chosen": -1.1658565998077393,
+      "logps/rejected": -1.406781554222107,
+      "loss": 1.2305,
+      "odds_ratio_loss": 0.6469210386276245,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.11658565700054169,
+      "rewards/margins": 0.024092501029372215,
+      "rewards/rejected": -0.14067815244197845,
+      "sft_loss": 1.1658565998077393,
+      "step": 1670
+    },
+    {
+      "epoch": 2.7157001414427158,
+      "grad_norm": 1.2988094091415405,
+      "learning_rate": 1.0789140688756805e-07,
+      "logits/chosen": -21.001827239990234,
+      "logits/rejected": -20.79971694946289,
+      "logps/chosen": -1.2181146144866943,
+      "logps/rejected": -1.434340238571167,
+      "loss": 1.2805,
+      "odds_ratio_loss": 0.6233752369880676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12181148678064346,
+      "rewards/margins": 0.0216225478798151,
+      "rewards/rejected": -0.14343401789665222,
+      "sft_loss": 1.2181146144866943,
+      "step": 1680
+    },
+    {
+      "epoch": 2.7318650232370176,
+      "grad_norm": 4.973554611206055,
+      "learning_rate": 9.592411742693098e-08,
+      "logits/chosen": -20.57479476928711,
+      "logits/rejected": -20.399307250976562,
+      "logps/chosen": -1.2814916372299194,
+      "logps/rejected": -1.3428285121917725,
+      "loss": 1.3583,
+      "odds_ratio_loss": 0.7676435112953186,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.12814916670322418,
+      "rewards/margins": 0.006133680697530508,
+      "rewards/rejected": -0.13428285717964172,
+      "sft_loss": 1.2814916372299194,
+      "step": 1690
+    },
+    {
+      "epoch": 2.7480299050313195,
+      "grad_norm": 1.3830405473709106,
+      "learning_rate": 8.464717095022168e-08,
+      "logits/chosen": -20.695613861083984,
+      "logits/rejected": -20.391788482666016,
+      "logps/chosen": -1.1915602684020996,
+      "logps/rejected": -1.3796567916870117,
+      "loss": 1.2587,
+      "odds_ratio_loss": 0.671554684638977,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.11915602535009384,
+      "rewards/margins": 0.01880965754389763,
+      "rewards/rejected": -0.13796567916870117,
+      "sft_loss": 1.1915602684020996,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7641947868256214,
+      "grad_norm": 2.5777227878570557,
+      "learning_rate": 7.406380569169841e-08,
+      "logits/chosen": -20.552873611450195,
+      "logits/rejected": -20.59903335571289,
+      "logps/chosen": -1.2640081644058228,
+      "logps/rejected": -1.334334373474121,
+      "loss": 1.3366,
+      "odds_ratio_loss": 0.7264189720153809,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.12640079855918884,
+      "rewards/margins": 0.0070326318964362144,
+      "rewards/rejected": -0.13343344628810883,
+      "sft_loss": 1.2640081644058228,
+      "step": 1710
+    },
+    {
+      "epoch": 2.7803596686199232,
+      "grad_norm": 10.019956588745117,
+      "learning_rate": 6.417706072013808e-08,
+      "logits/chosen": -20.83199119567871,
+      "logits/rejected": -21.08270263671875,
+      "logps/chosen": -1.1969940662384033,
+      "logps/rejected": -1.326841115951538,
+      "loss": 1.2649,
+      "odds_ratio_loss": 0.6789032816886902,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11969941854476929,
+      "rewards/margins": 0.012984703294932842,
+      "rewards/rejected": -0.1326841115951538,
+      "sft_loss": 1.1969940662384033,
+      "step": 1720
+    },
+    {
+      "epoch": 2.796524550414225,
+      "grad_norm": 6.8160624504089355,
+      "learning_rate": 5.498977506615294e-08,
+      "logits/chosen": -20.777956008911133,
+      "logits/rejected": -20.86117172241211,
+      "logps/chosen": -1.2800042629241943,
+      "logps/rejected": -1.3593509197235107,
+      "loss": 1.3517,
+      "odds_ratio_loss": 0.7168198227882385,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1280004382133484,
+      "rewards/margins": 0.007934670895338058,
+      "rewards/rejected": -0.13593511283397675,
+      "sft_loss": 1.2800042629241943,
+      "step": 1730
+    },
+    {
+      "epoch": 2.812689432208527,
+      "grad_norm": 1.6938509941101074,
+      "learning_rate": 4.6504586906947756e-08,
+      "logits/chosen": -21.01276206970215,
+      "logits/rejected": -21.02105140686035,
+      "logps/chosen": -1.2667747735977173,
+      "logps/rejected": -1.3524465560913086,
+      "loss": 1.3367,
+      "odds_ratio_loss": 0.6990799903869629,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.126677468419075,
+      "rewards/margins": 0.008567174896597862,
+      "rewards/rejected": -0.13524465262889862,
+      "sft_loss": 1.2667747735977173,
+      "step": 1740
+    },
+    {
+      "epoch": 2.828854314002829,
+      "grad_norm": 6.788274765014648,
+      "learning_rate": 3.8723932808754914e-08,
+      "logits/chosen": -21.133237838745117,
+      "logits/rejected": -21.116573333740234,
+      "logps/chosen": -1.3402550220489502,
+      "logps/rejected": -1.3987007141113281,
+      "loss": 1.4126,
+      "odds_ratio_loss": 0.7239315509796143,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.13402549922466278,
+      "rewards/margins": 0.005844591651111841,
+      "rewards/rejected": -0.1398700773715973,
+      "sft_loss": 1.3402550220489502,
+      "step": 1750
+    },
+    {
+      "epoch": 2.8450191957971307,
+      "grad_norm": 4.773237228393555,
+      "learning_rate": 3.1650047027158014e-08,
+      "logits/chosen": -20.84109115600586,
+      "logits/rejected": -21.033550262451172,
+      "logps/chosen": -1.2243554592132568,
+      "logps/rejected": -1.3495168685913086,
+      "loss": 1.2905,
+      "odds_ratio_loss": 0.6615921258926392,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.12243553251028061,
+      "rewards/margins": 0.012516150251030922,
+      "rewards/rejected": -0.13495168089866638,
+      "sft_loss": 1.2243554592132568,
+      "step": 1760
+    },
+    {
+      "epoch": 2.8611840775914326,
+      "grad_norm": 1.8300281763076782,
+      "learning_rate": 2.5284960865517848e-08,
+      "logits/chosen": -20.63343048095703,
+      "logits/rejected": -20.709888458251953,
+      "logps/chosen": -1.1276973485946655,
+      "logps/rejected": -1.3769396543502808,
+      "loss": 1.1906,
+      "odds_ratio_loss": 0.6287329196929932,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.11276974529027939,
+      "rewards/margins": 0.024924224242568016,
+      "rewards/rejected": -0.13769397139549255,
+      "sft_loss": 1.1276973485946655,
+      "step": 1770
+    },
+    {
+      "epoch": 2.8773489593857344,
+      "grad_norm": 1.8408336639404297,
+      "learning_rate": 1.9630502091670388e-08,
+      "logits/chosen": -20.521255493164062,
+      "logits/rejected": -20.47439956665039,
+      "logps/chosen": -1.2427160739898682,
+      "logps/rejected": -1.4044514894485474,
+      "loss": 1.3062,
+      "odds_ratio_loss": 0.6351091265678406,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12427163124084473,
+      "rewards/margins": 0.016173530369997025,
+      "rewards/rejected": -0.14044514298439026,
+      "sft_loss": 1.2427160739898682,
+      "step": 1780
+    },
+    {
+      "epoch": 2.8935138411800363,
+      "grad_norm": 5.715972423553467,
+      "learning_rate": 1.4688294413074677e-08,
+      "logits/chosen": -20.392711639404297,
+      "logits/rejected": -20.409814834594727,
+      "logps/chosen": -1.12639582157135,
+      "logps/rejected": -1.3582656383514404,
+      "loss": 1.193,
+      "odds_ratio_loss": 0.6661497354507446,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11263958364725113,
+      "rewards/margins": 0.023186981678009033,
+      "rewards/rejected": -0.13582655787467957,
+      "sft_loss": 1.12639582157135,
+      "step": 1790
+    },
+    {
+      "epoch": 2.909678722974338,
+      "grad_norm": 1.500075340270996,
+      "learning_rate": 1.0459757010556626e-08,
+      "logits/chosen": -20.747310638427734,
+      "logits/rejected": -20.764245986938477,
+      "logps/chosen": -1.218435287475586,
+      "logps/rejected": -1.2865406274795532,
+      "loss": 1.2909,
+      "odds_ratio_loss": 0.724606990814209,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.12184351682662964,
+      "rewards/margins": 0.0068105412647128105,
+      "rewards/rejected": -0.12865407764911652,
+      "sft_loss": 1.218435287475586,
+      "step": 1800
+    },
+    {
+      "epoch": 2.92584360476864,
+      "grad_norm": 4.831416606903076,
+      "learning_rate": 6.94610413078306e-09,
+      "logits/chosen": -20.301036834716797,
+      "logits/rejected": -20.699962615966797,
+      "logps/chosen": -1.2750685214996338,
+      "logps/rejected": -1.4494614601135254,
+      "loss": 1.3451,
+      "odds_ratio_loss": 0.7007848024368286,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12750686705112457,
+      "rewards/margins": 0.017439283430576324,
+      "rewards/rejected": -0.1449461281299591,
+      "sft_loss": 1.2750685214996338,
+      "step": 1810
+    },
+    {
+      "epoch": 2.942008486562942,
+      "grad_norm": 3.175490617752075,
+      "learning_rate": 4.14834473758563e-09,
+      "logits/chosen": -20.20795249938965,
+      "logits/rejected": -20.349489212036133,
+      "logps/chosen": -1.168878197669983,
+      "logps/rejected": -1.3752342462539673,
+      "loss": 1.2352,
+      "odds_ratio_loss": 0.6636675596237183,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.11688783019781113,
+      "rewards/margins": 0.020635612308979034,
+      "rewards/rejected": -0.13752342760562897,
+      "sft_loss": 1.168878197669983,
+      "step": 1820
+    },
+    {
+      "epoch": 2.9581733683572438,
+      "grad_norm": 1.267177700996399,
+      "learning_rate": 2.067282222230349e-09,
+      "logits/chosen": -20.883831024169922,
+      "logits/rejected": -21.049150466918945,
+      "logps/chosen": -1.19767427444458,
+      "logps/rejected": -1.4529333114624023,
+      "loss": 1.2618,
+      "odds_ratio_loss": 0.6415713429450989,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1197674423456192,
+      "rewards/margins": 0.02552589774131775,
+      "rewards/rejected": -0.14529332518577576,
+      "sft_loss": 1.19767427444458,
+      "step": 1830
+    },
+    {
+      "epoch": 2.9743382501515456,
+      "grad_norm": 6.299037456512451,
+      "learning_rate": 7.035141727212979e-10,
+      "logits/chosen": -20.69939422607422,
+      "logits/rejected": -20.93973159790039,
+      "logps/chosen": -1.1146414279937744,
+      "logps/rejected": -1.2947094440460205,
+      "loss": 1.1819,
+      "odds_ratio_loss": 0.6721066236495972,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11146414279937744,
+      "rewards/margins": 0.018006790429353714,
+      "rewards/rejected": -0.12947094440460205,
+      "sft_loss": 1.1146414279937744,
+      "step": 1840
+    },
+    {
+      "epoch": 2.9905031319458475,
+      "grad_norm": 7.900111675262451,
+      "learning_rate": 5.743220219761592e-11,
+      "logits/chosen": -20.688940048217773,
+      "logits/rejected": -20.652080535888672,
+      "logps/chosen": -1.2903720140457153,
+      "logps/rejected": -1.3897130489349365,
+      "loss": 1.3642,
+      "odds_ratio_loss": 0.7377926111221313,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12903721630573273,
+      "rewards/margins": 0.00993410311639309,
+      "rewards/rejected": -0.13897129893302917,
+      "sft_loss": 1.2903720140457153,
+      "step": 1850
+    },
+    {
+      "epoch": 2.9969690846635686,
+      "step": 1854,
+      "total_flos": 5.618252880760013e+17,
+      "train_loss": 1.3892074464594277,
+      "train_runtime": 8722.5011,
+      "train_samples_per_second": 3.404,
+      "train_steps_per_second": 0.213
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1854,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 5.618252880760013e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}