{
  "best_metric": 1.3293771743774414,
  "best_model_checkpoint": "saves/Gemma-2B-It/lora/orpo-salt/checkpoint-1500",
  "epoch": 2.9969690846635686,
  "eval_steps": 500,
  "global_step": 1854,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01616488179430188,
      "grad_norm": 3.7809898853302,
      "learning_rate": 4.999648198770648e-06,
      "logits/chosen": -21.99495506286621,
      "logits/rejected": -22.17684555053711,
      "logps/chosen": -2.1064062118530273,
      "logps/rejected": -2.4076905250549316,
      "loss": 2.18,
      "odds_ratio_loss": 0.7356098294258118,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.21064062416553497,
      "rewards/margins": 0.03012842871248722,
      "rewards/rejected": -0.24076907336711884,
      "sft_loss": 2.1064062118530273,
      "step": 10
    },
    {
      "epoch": 0.03232976358860376,
      "grad_norm": 2.3069310188293457,
      "learning_rate": 4.998578646361359e-06,
      "logits/chosen": -21.845182418823242,
      "logits/rejected": -21.866207122802734,
      "logps/chosen": -2.080873489379883,
      "logps/rejected": -2.124648332595825,
      "loss": 2.1663,
      "odds_ratio_loss": 0.8539472818374634,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.20808735489845276,
      "rewards/margins": 0.004377496428787708,
      "rewards/rejected": -0.2124648541212082,
      "sft_loss": 2.080873489379883,
      "step": 20
    },
    {
      "epoch": 0.04849464538290564,
      "grad_norm": 5.5587544441223145,
      "learning_rate": 4.996791614004449e-06,
      "logits/chosen": -21.90616226196289,
      "logits/rejected": -21.923837661743164,
      "logps/chosen": -2.2512574195861816,
      "logps/rejected": -2.225501537322998,
      "loss": 2.3454,
      "odds_ratio_loss": 0.9413110017776489,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.22512571513652802,
      "rewards/margins": -0.0025755497626960278,
      "rewards/rejected": -0.2225501835346222,
      "sft_loss": 2.2512574195861816,
      "step": 30
    },
    {
      "epoch": 0.06465952717720752,
      "grad_norm": 3.6638951301574707,
      "learning_rate": 4.994287614855618e-06,
      "logits/chosen": -21.935110092163086,
      "logits/rejected": -22.104324340820312,
      "logps/chosen": -2.171135425567627,
      "logps/rejected": -2.2752678394317627,
      "loss": 2.267,
      "odds_ratio_loss": 0.9586257934570312,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.21711353957653046,
      "rewards/margins": 0.010413245297968388,
      "rewards/rejected": -0.22752678394317627,
      "sft_loss": 2.171135425567627,
      "step": 40
    },
    {
      "epoch": 0.0808244089715094,
      "grad_norm": 6.454894065856934,
      "learning_rate": 4.991067367951343e-06,
      "logits/chosen": -22.541332244873047,
      "logits/rejected": -22.472043991088867,
      "logps/chosen": -2.1005358695983887,
      "logps/rejected": -2.1757073402404785,
      "loss": 2.1825,
      "odds_ratio_loss": 0.8198438882827759,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21005356311798096,
      "rewards/margins": 0.007517138961702585,
      "rewards/rejected": -0.2175707072019577,
      "sft_loss": 2.1005358695983887,
      "step": 50
    },
    {
      "epoch": 0.09698929076581128,
      "grad_norm": 2.4648330211639404,
      "learning_rate": 4.987131798002389e-06,
      "logits/chosen": -22.33713722229004,
      "logits/rejected": -22.142881393432617,
      "logps/chosen": -2.0364699363708496,
      "logps/rejected": -2.22188138961792,
      "loss": 2.1252,
      "odds_ratio_loss": 0.887610912322998,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.20364701747894287,
      "rewards/margins": 0.018541133031249046,
      "rewards/rejected": -0.22218814492225647,
      "sft_loss": 2.0364699363708496,
      "step": 60
    },
    {
      "epoch": 0.11315417256011315,
      "grad_norm": 4.808262825012207,
      "learning_rate": 4.982482035128285e-06,
      "logits/chosen": -22.240699768066406,
      "logits/rejected": -22.348970413208008,
      "logps/chosen": -2.144533634185791,
      "logps/rejected": -2.3089094161987305,
      "loss": 2.2393,
      "odds_ratio_loss": 0.9479582905769348,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.21445336937904358,
      "rewards/margins": 0.01643756404519081,
      "rewards/rejected": -0.23089094460010529,
      "sft_loss": 2.144533634185791,
      "step": 70
    },
    {
      "epoch": 0.12931905435441504,
      "grad_norm": 5.7843122482299805,
      "learning_rate": 4.9771194145328e-06,
      "logits/chosen": -22.19432258605957,
      "logits/rejected": -22.165550231933594,
      "logps/chosen": -1.7035343647003174,
      "logps/rejected": -2.021710157394409,
      "loss": 1.7739,
      "odds_ratio_loss": 0.7040209174156189,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.17035342752933502,
      "rewards/margins": 0.03181760013103485,
      "rewards/rejected": -0.20217104256153107,
      "sft_loss": 1.7035343647003174,
      "step": 80
    },
    {
      "epoch": 0.1454839361487169,
      "grad_norm": 10.499777793884277,
      "learning_rate": 4.971045476120532e-06,
      "logits/chosen": -22.63768196105957,
      "logits/rejected": -22.5053768157959,
      "logps/chosen": -1.7764383554458618,
      "logps/rejected": -1.8638198375701904,
      "loss": 1.8559,
      "odds_ratio_loss": 0.7946864366531372,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.17764383554458618,
      "rewards/margins": 0.008738175965845585,
      "rewards/rejected": -0.1863820105791092,
      "sft_loss": 1.7764383554458618,
      "step": 90
    },
    {
      "epoch": 0.1616488179430188,
      "grad_norm": 2.4865188598632812,
      "learning_rate": 4.964261964054713e-06,
      "logits/chosen": -22.539587020874023,
      "logits/rejected": -22.56546401977539,
      "logps/chosen": -1.846840262413025,
      "logps/rejected": -2.0120983123779297,
      "loss": 1.9298,
      "odds_ratio_loss": 0.829124927520752,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.18468405306339264,
      "rewards/margins": 0.016525788232684135,
      "rewards/rejected": -0.20120985805988312,
      "sft_loss": 1.846840262413025,
      "step": 100
    },
    {
      "epoch": 0.17781369973732067,
      "grad_norm": 8.431103706359863,
      "learning_rate": 4.956770826256372e-06,
      "logits/chosen": -22.575443267822266,
      "logits/rejected": -22.463134765625,
      "logps/chosen": -1.6286426782608032,
      "logps/rejected": -1.7305223941802979,
      "loss": 1.7064,
      "odds_ratio_loss": 0.7774398922920227,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.16286428272724152,
      "rewards/margins": 0.010187968611717224,
      "rewards/rejected": -0.17305223643779755,
      "sft_loss": 1.6286426782608032,
      "step": 110
    },
    {
      "epoch": 0.19397858153162256,
      "grad_norm": 2.32936429977417,
      "learning_rate": 4.94857421384497e-06,
      "logits/chosen": -22.672468185424805,
      "logits/rejected": -22.749752044677734,
      "logps/chosen": -1.6404567956924438,
      "logps/rejected": -1.8715522289276123,
      "loss": 1.7191,
      "odds_ratio_loss": 0.7863814830780029,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.16404569149017334,
      "rewards/margins": 0.023109545931220055,
      "rewards/rejected": -0.18715524673461914,
      "sft_loss": 1.6404567956924438,
      "step": 120
    },
    {
      "epoch": 0.21014346332592443,
      "grad_norm": 2.062532663345337,
      "learning_rate": 4.939674480520701e-06,
      "logits/chosen": -22.727245330810547,
      "logits/rejected": -22.860414505004883,
      "logps/chosen": -1.5943523645401,
      "logps/rejected": -1.7242279052734375,
      "loss": 1.6786,
      "odds_ratio_loss": 0.8422672152519226,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1594352275133133,
      "rewards/margins": 0.012987568974494934,
      "rewards/rejected": -0.17242279648780823,
      "sft_loss": 1.5943523645401,
      "step": 130
    },
    {
      "epoch": 0.2263083451202263,
      "grad_norm": 1.7461190223693848,
      "learning_rate": 4.930074181888613e-06,
      "logits/chosen": -22.773456573486328,
      "logits/rejected": -22.877899169921875,
      "logps/chosen": -1.680092453956604,
      "logps/rejected": -1.8227007389068604,
      "loss": 1.7527,
      "odds_ratio_loss": 0.7260502576828003,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.16800925135612488,
      "rewards/margins": 0.014260831288993359,
      "rewards/rejected": -0.1822700798511505,
      "sft_loss": 1.680092453956604,
      "step": 140
    },
    {
      "epoch": 0.2424732269145282,
      "grad_norm": 3.3798775672912598,
      "learning_rate": 4.91977607472475e-06,
      "logits/chosen": -23.01150131225586,
      "logits/rejected": -22.988527297973633,
      "logps/chosen": -1.5780761241912842,
      "logps/rejected": -1.6528291702270508,
      "loss": 1.6563,
      "odds_ratio_loss": 0.7826321721076965,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.1578076183795929,
      "rewards/margins": 0.007475291378796101,
      "rewards/rejected": -0.16528290510177612,
      "sft_loss": 1.5780761241912842,
      "step": 150
    },
    {
      "epoch": 0.2586381087088301,
      "grad_norm": 1.8430626392364502,
      "learning_rate": 4.908783116184534e-06,
      "logits/chosen": -22.70990562438965,
      "logits/rejected": -22.700252532958984,
      "logps/chosen": -1.5454621315002441,
      "logps/rejected": -1.7210956811904907,
      "loss": 1.614,
      "odds_ratio_loss": 0.6857101917266846,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.15454623103141785,
      "rewards/margins": 0.017563361674547195,
      "rewards/rejected": -0.17210958898067474,
      "sft_loss": 1.5454621315002441,
      "step": 160
    },
    {
      "epoch": 0.27480299050313195,
      "grad_norm": 1.635880470275879,
      "learning_rate": 4.897098462953598e-06,
      "logits/chosen": -23.044336318969727,
      "logits/rejected": -23.024044036865234,
      "logps/chosen": -1.457194447517395,
      "logps/rejected": -1.7030904293060303,
      "loss": 1.5297,
      "odds_ratio_loss": 0.7254581451416016,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.14571943879127502,
      "rewards/margins": 0.02458963170647621,
      "rewards/rejected": -0.17030906677246094,
      "sft_loss": 1.457194447517395,
      "step": 170
    },
    {
      "epoch": 0.2909678722974338,
      "grad_norm": 1.3249585628509521,
      "learning_rate": 4.884725470341331e-06,
      "logits/chosen": -23.00472640991211,
      "logits/rejected": -23.162250518798828,
      "logps/chosen": -1.3314763307571411,
      "logps/rejected": -1.5617011785507202,
      "loss": 1.3961,
      "odds_ratio_loss": 0.6465403437614441,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.13314762711524963,
      "rewards/margins": 0.023022485896945,
      "rewards/rejected": -0.15617012977600098,
      "sft_loss": 1.3314763307571411,
      "step": 180
    },
    {
      "epoch": 0.3071327540917357,
      "grad_norm": 4.37128210067749,
      "learning_rate": 4.871667691317377e-06,
      "logits/chosen": -23.25027847290039,
      "logits/rejected": -23.159801483154297,
      "logps/chosen": -1.5023282766342163,
      "logps/rejected": -1.5170981884002686,
      "loss": 1.5883,
      "odds_ratio_loss": 0.8601799011230469,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.15023282170295715,
      "rewards/margins": 0.0014769884292036295,
      "rewards/rejected": -0.15170980989933014,
      "sft_loss": 1.5023282766342163,
      "step": 190
    },
    {
      "epoch": 0.3232976358860376,
      "grad_norm": 12.613110542297363,
      "learning_rate": 4.857928875491392e-06,
      "logits/chosen": -23.007986068725586,
      "logits/rejected": -22.91147232055664,
      "logps/chosen": -1.360762596130371,
      "logps/rejected": -1.479021668434143,
      "loss": 1.4383,
      "odds_ratio_loss": 0.7751696705818176,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13607625663280487,
      "rewards/margins": 0.01182589866220951,
      "rewards/rejected": -0.14790216088294983,
      "sft_loss": 1.360762596130371,
      "step": 200
    },
    {
      "epoch": 0.33946251768033947,
      "grad_norm": 2.2189383506774902,
      "learning_rate": 4.843512968036314e-06,
      "logits/chosen": -22.693614959716797,
      "logits/rejected": -22.657732009887695,
      "logps/chosen": -1.3642535209655762,
      "logps/rejected": -1.4226882457733154,
      "loss": 1.4427,
      "odds_ratio_loss": 0.7848214507102966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13642534613609314,
      "rewards/margins": 0.0058434633538126945,
      "rewards/rejected": -0.1422688066959381,
      "sft_loss": 1.3642535209655762,
      "step": 210
    },
    {
      "epoch": 0.35562739947464134,
      "grad_norm": 4.814913749694824,
      "learning_rate": 4.828424108555486e-06,
      "logits/chosen": -23.161609649658203,
      "logits/rejected": -23.108890533447266,
      "logps/chosen": -1.5788774490356445,
      "logps/rejected": -1.7273142337799072,
      "loss": 1.6537,
      "odds_ratio_loss": 0.748489499092102,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.15788774192333221,
      "rewards/margins": 0.014843670651316643,
      "rewards/rejected": -0.1727314293384552,
      "sft_loss": 1.5788774490356445,
      "step": 220
    },
    {
      "epoch": 0.3717922812689432,
      "grad_norm": 0.8914692997932434,
      "learning_rate": 4.812666629893957e-06,
      "logits/chosen": -23.069910049438477,
      "logits/rejected": -23.09903907775879,
      "logps/chosen": -1.3772187232971191,
      "logps/rejected": -1.4173983335494995,
      "loss": 1.4523,
      "odds_ratio_loss": 0.7508037686347961,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13772186636924744,
      "rewards/margins": 0.004017965402454138,
      "rewards/rejected": -0.1417398303747177,
      "sft_loss": 1.3772187232971191,
      "step": 230
    },
    {
      "epoch": 0.3879571630632451,
      "grad_norm": 1.202868103981018,
      "learning_rate": 4.796245056894273e-06,
      "logits/chosen": -22.570789337158203,
      "logits/rejected": -22.600833892822266,
      "logps/chosen": -1.453567385673523,
      "logps/rejected": -1.5844666957855225,
      "loss": 1.5321,
      "odds_ratio_loss": 0.7851130962371826,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.14535674452781677,
      "rewards/margins": 0.013089925050735474,
      "rewards/rejected": -0.15844666957855225,
      "sft_loss": 1.453567385673523,
      "step": 240
    },
    {
      "epoch": 0.404122044857547,
      "grad_norm": 1.3130539655685425,
      "learning_rate": 4.779164105097148e-06,
      "logits/chosen": -23.004568099975586,
      "logits/rejected": -23.11286163330078,
      "logps/chosen": -1.325037956237793,
      "logps/rejected": -1.563007116317749,
      "loss": 1.391,
      "odds_ratio_loss": 0.6598650813102722,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13250377774238586,
      "rewards/margins": 0.023796914145350456,
      "rewards/rejected": -0.15630069375038147,
      "sft_loss": 1.325037956237793,
      "step": 250
    },
    {
      "epoch": 0.42028692665184886,
      "grad_norm": 1.867422342300415,
      "learning_rate": 4.761428679387373e-06,
      "logits/chosen": -22.629825592041016,
      "logits/rejected": -22.818918228149414,
      "logps/chosen": -1.317091941833496,
      "logps/rejected": -1.4807102680206299,
      "loss": 1.3842,
      "odds_ratio_loss": 0.6714745759963989,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13170920312404633,
      "rewards/margins": 0.016361821442842484,
      "rewards/rejected": -0.1480710208415985,
      "sft_loss": 1.317091941833496,
      "step": 260
    },
    {
      "epoch": 0.4364518084461507,
      "grad_norm": 1.9516927003860474,
      "learning_rate": 4.7430438725853515e-06,
      "logits/chosen": -22.51862907409668,
      "logits/rejected": -22.682025909423828,
      "logps/chosen": -1.4145148992538452,
      "logps/rejected": -1.653055191040039,
      "loss": 1.4844,
      "odds_ratio_loss": 0.6991928815841675,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.14145150780677795,
      "rewards/margins": 0.023854022845625877,
      "rewards/rejected": -0.16530552506446838,
      "sft_loss": 1.4145148992538452,
      "step": 270
    },
    {
      "epoch": 0.4526166902404526,
      "grad_norm": 1.7087641954421997,
      "learning_rate": 4.724014963984669e-06,
      "logits/chosen": -22.849842071533203,
      "logits/rejected": -22.771404266357422,
      "logps/chosen": -1.387978196144104,
      "logps/rejected": -1.5646307468414307,
      "loss": 1.4589,
      "odds_ratio_loss": 0.7090586423873901,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.1387978196144104,
      "rewards/margins": 0.01766524836421013,
      "rewards/rejected": -0.15646305680274963,
      "sft_loss": 1.387978196144104,
      "step": 280
    },
    {
      "epoch": 0.4687815720347545,
      "grad_norm": 3.348365068435669,
      "learning_rate": 4.704347417836116e-06,
      "logits/chosen": -22.53603172302246,
      "logits/rejected": -22.64337730407715,
      "logps/chosen": -1.2867505550384521,
      "logps/rejected": -1.4700194597244263,
      "loss": 1.3571,
      "odds_ratio_loss": 0.7038428783416748,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12867505848407745,
      "rewards/margins": 0.018326906487345695,
      "rewards/rejected": -0.1470019519329071,
      "sft_loss": 1.2867505550384521,
      "step": 290
    },
    {
      "epoch": 0.4849464538290564,
      "grad_norm": 2.8705925941467285,
      "learning_rate": 4.684046881778603e-06,
      "logits/chosen": -22.37331771850586,
      "logits/rejected": -22.327777862548828,
      "logps/chosen": -1.3365970849990845,
      "logps/rejected": -1.4279487133026123,
      "loss": 1.4075,
      "odds_ratio_loss": 0.7086235880851746,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13365969061851501,
      "rewards/margins": 0.009135158732533455,
      "rewards/rejected": -0.14279486238956451,
      "sft_loss": 1.3365970849990845,
      "step": 300
    },
    {
      "epoch": 0.5011113356233583,
      "grad_norm": 1.1998392343521118,
      "learning_rate": 4.663119185217409e-06,
      "logits/chosen": -22.524127960205078,
      "logits/rejected": -22.61183738708496,
      "logps/chosen": -1.3241046667099,
      "logps/rejected": -1.48367440700531,
      "loss": 1.3928,
      "odds_ratio_loss": 0.6864575147628784,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13241048157215118,
      "rewards/margins": 0.015956982970237732,
      "rewards/rejected": -0.14836746454238892,
      "sft_loss": 1.3241046667099,
      "step": 310
    },
    {
      "epoch": 0.5172762174176602,
      "grad_norm": 1.1742535829544067,
      "learning_rate": 4.641570337650232e-06,
      "logits/chosen": -22.668704986572266,
      "logits/rejected": -22.804088592529297,
      "logps/chosen": -1.2428923845291138,
      "logps/rejected": -1.3876885175704956,
      "loss": 1.3138,
      "odds_ratio_loss": 0.7085933685302734,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12428925186395645,
      "rewards/margins": 0.014479607343673706,
      "rewards/rejected": -0.13876885175704956,
      "sft_loss": 1.2428923845291138,
      "step": 320
    },
    {
      "epoch": 0.533441099211962,
      "grad_norm": 3.9174962043762207,
      "learning_rate": 4.61940652694154e-06,
      "logits/chosen": -22.036165237426758,
      "logits/rejected": -22.13457679748535,
      "logps/chosen": -1.3345186710357666,
      "logps/rejected": -1.4239206314086914,
      "loss": 1.4082,
      "odds_ratio_loss": 0.7365013360977173,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.13345186412334442,
      "rewards/margins": 0.008940198458731174,
      "rewards/rejected": -0.14239206910133362,
      "sft_loss": 1.3345186710357666,
      "step": 330
    },
    {
      "epoch": 0.5496059810062639,
      "grad_norm": 3.5989551544189453,
      "learning_rate": 4.596634117545689e-06,
      "logits/chosen": -22.659748077392578,
      "logits/rejected": -22.487016677856445,
      "logps/chosen": -1.4412176609039307,
      "logps/rejected": -1.6016347408294678,
      "loss": 1.5117,
      "odds_ratio_loss": 0.7047333121299744,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.14412176609039307,
      "rewards/margins": 0.016041725873947144,
      "rewards/rejected": -0.1601634919643402,
      "sft_loss": 1.4412176609039307,
      "step": 340
    },
    {
      "epoch": 0.5657708628005658,
      "grad_norm": 1.72525954246521,
      "learning_rate": 4.573259648679335e-06,
      "logits/chosen": -22.422473907470703,
      "logits/rejected": -22.158262252807617,
      "logps/chosen": -1.3891839981079102,
      "logps/rejected": -1.5595767498016357,
      "loss": 1.4614,
      "odds_ratio_loss": 0.7222477197647095,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13891839981079102,
      "rewards/margins": 0.017039284110069275,
      "rewards/rejected": -0.1559576839208603,
      "sft_loss": 1.3891839981079102,
      "step": 350
    },
    {
      "epoch": 0.5819357445948676,
      "grad_norm": 6.434234619140625,
      "learning_rate": 4.549289832443663e-06,
      "logits/chosen": -22.174190521240234,
      "logits/rejected": -22.23097038269043,
      "logps/chosen": -1.315002202987671,
      "logps/rejected": -1.472235918045044,
      "loss": 1.3873,
      "odds_ratio_loss": 0.7233031988143921,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1315002143383026,
      "rewards/margins": 0.015723396092653275,
      "rewards/rejected": -0.14722362160682678,
      "sft_loss": 1.315002202987671,
      "step": 360
    },
    {
      "epoch": 0.5981006263891695,
      "grad_norm": 1.595171332359314,
      "learning_rate": 4.524731551896978e-06,
      "logits/chosen": -21.987152099609375,
      "logits/rejected": -21.909650802612305,
      "logps/chosen": -1.2452141046524048,
      "logps/rejected": -1.389278769493103,
      "loss": 1.3152,
      "odds_ratio_loss": 0.7003225088119507,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.1245214194059372,
      "rewards/margins": 0.014406457543373108,
      "rewards/rejected": -0.1389278769493103,
      "sft_loss": 1.2452141046524048,
      "step": 370
    },
    {
      "epoch": 0.6142655081834714,
      "grad_norm": 2.505262851715088,
      "learning_rate": 4.4995918590781925e-06,
      "logits/chosen": -22.261516571044922,
      "logits/rejected": -22.25895118713379,
      "logps/chosen": -1.3019020557403564,
      "logps/rejected": -1.4299681186676025,
      "loss": 1.3751,
      "odds_ratio_loss": 0.7319445610046387,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13019022345542908,
      "rewards/margins": 0.012806607410311699,
      "rewards/rejected": -0.14299681782722473,
      "sft_loss": 1.3019020557403564,
      "step": 380
    },
    {
      "epoch": 0.6304303899777733,
      "grad_norm": 1.0513848066329956,
      "learning_rate": 4.473877972981797e-06,
      "logits/chosen": -21.957128524780273,
      "logits/rejected": -21.8829288482666,
      "logps/chosen": -1.343338131904602,
      "logps/rejected": -1.5255149602890015,
      "loss": 1.4114,
      "odds_ratio_loss": 0.6804416179656982,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13433381915092468,
      "rewards/margins": 0.01821768842637539,
      "rewards/rejected": -0.15255150198936462,
      "sft_loss": 1.343338131904602,
      "step": 390
    },
    {
      "epoch": 0.6465952717720752,
      "grad_norm": 2.7681405544281006,
      "learning_rate": 4.447597277484894e-06,
      "logits/chosen": -21.926578521728516,
      "logits/rejected": -21.960134506225586,
      "logps/chosen": -1.2327673435211182,
      "logps/rejected": -1.3560773134231567,
      "loss": 1.305,
      "odds_ratio_loss": 0.7222930192947388,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.1232767328619957,
      "rewards/margins": 0.012331007048487663,
      "rewards/rejected": -0.1356077492237091,
      "sft_loss": 1.2327673435211182,
      "step": 400
    },
    {
      "epoch": 0.6627601535663771,
      "grad_norm": 1.4995496273040771,
      "learning_rate": 4.42075731922687e-06,
      "logits/chosen": -22.4599552154541,
      "logits/rejected": -22.27519989013672,
      "logps/chosen": -1.383239507675171,
      "logps/rejected": -1.4447576999664307,
      "loss": 1.4576,
      "odds_ratio_loss": 0.7433907389640808,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13832393288612366,
      "rewards/margins": 0.006151811219751835,
      "rewards/rejected": -0.1444757580757141,
      "sft_loss": 1.383239507675171,
      "step": 410
    },
    {
      "epoch": 0.6789250353606789,
      "grad_norm": 3.4521114826202393,
      "learning_rate": 4.3933658054423465e-06,
      "logits/chosen": -22.025806427001953,
      "logits/rejected": -21.983091354370117,
      "logps/chosen": -1.2636308670043945,
      "logps/rejected": -1.3890178203582764,
      "loss": 1.3327,
      "odds_ratio_loss": 0.6908599138259888,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12636308372020721,
      "rewards/margins": 0.012538706883788109,
      "rewards/rejected": -0.13890179991722107,
      "sft_loss": 1.2636308670043945,
      "step": 420
    },
    {
      "epoch": 0.6950899171549808,
      "grad_norm": 2.2511396408081055,
      "learning_rate": 4.365430601748003e-06,
      "logits/chosen": -21.880239486694336,
      "logits/rejected": -22.0391845703125,
      "logps/chosen": -1.4012739658355713,
      "logps/rejected": -1.4651644229888916,
      "loss": 1.4754,
      "odds_ratio_loss": 0.7415169477462769,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14012740552425385,
      "rewards/margins": 0.006389034446328878,
      "rewards/rejected": -0.14651645720005035,
      "sft_loss": 1.4012739658355713,
      "step": 430
    },
    {
      "epoch": 0.7112547989492827,
      "grad_norm": 6.550912857055664,
      "learning_rate": 4.336959729883925e-06,
      "logits/chosen": -22.025110244750977,
      "logits/rejected": -21.944046020507812,
      "logps/chosen": -1.2905068397521973,
      "logps/rejected": -1.3223841190338135,
      "loss": 1.3664,
      "odds_ratio_loss": 0.7585982084274292,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.12905070185661316,
      "rewards/margins": 0.003187725320458412,
      "rewards/rejected": -0.13223841786384583,
      "sft_loss": 1.2905068397521973,
      "step": 440
    },
    {
      "epoch": 0.7274196807435845,
      "grad_norm": 1.741584062576294,
      "learning_rate": 4.307961365410118e-06,
      "logits/chosen": -21.6479434967041,
      "logits/rejected": -21.64910888671875,
      "logps/chosen": -1.3282034397125244,
      "logps/rejected": -1.3957862854003906,
      "loss": 1.4023,
      "odds_ratio_loss": 0.7411014437675476,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.13282035291194916,
      "rewards/margins": 0.006758278701454401,
      "rewards/rejected": -0.13957861065864563,
      "sft_loss": 1.3282034397125244,
      "step": 450
    },
    {
      "epoch": 0.7435845625378864,
      "grad_norm": 2.630305528640747,
      "learning_rate": 4.278443835358854e-06,
      "logits/chosen": -22.009098052978516,
      "logits/rejected": -21.944019317626953,
      "logps/chosen": -1.2518932819366455,
      "logps/rejected": -1.4686853885650635,
      "loss": 1.3164,
      "odds_ratio_loss": 0.6453157067298889,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12518933415412903,
      "rewards/margins": 0.0216792281717062,
      "rewards/rejected": -0.14686855673789978,
      "sft_loss": 1.2518932819366455,
      "step": 460
    },
    {
      "epoch": 0.7597494443321883,
      "grad_norm": 1.7374227046966553,
      "learning_rate": 4.248415615843523e-06,
      "logits/chosen": -22.067617416381836,
      "logits/rejected": -21.912487030029297,
      "logps/chosen": -1.278611421585083,
      "logps/rejected": -1.367891550064087,
      "loss": 1.3508,
      "odds_ratio_loss": 0.7223629951477051,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1278611272573471,
      "rewards/margins": 0.008928016759455204,
      "rewards/rejected": -0.13678915798664093,
      "sft_loss": 1.278611421585083,
      "step": 470
    },
    {
      "epoch": 0.7759143261264903,
      "grad_norm": 5.568856716156006,
      "learning_rate": 4.217885329624666e-06,
      "logits/chosen": -22.169946670532227,
      "logits/rejected": -22.093523025512695,
      "logps/chosen": -1.207793951034546,
      "logps/rejected": -1.4276247024536133,
      "loss": 1.2738,
      "odds_ratio_loss": 0.660278856754303,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1207793727517128,
      "rewards/margins": 0.021983083337545395,
      "rewards/rejected": -0.1427624672651291,
      "sft_loss": 1.207793951034546,
      "step": 480
    },
    {
      "epoch": 0.7920792079207921,
      "grad_norm": 1.7587084770202637,
      "learning_rate": 4.186861743633911e-06,
      "logits/chosen": -21.444040298461914,
      "logits/rejected": -21.620454788208008,
      "logps/chosen": -1.2591201066970825,
      "logps/rejected": -1.494167685508728,
      "loss": 1.3277,
      "odds_ratio_loss": 0.6855591535568237,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.12591201066970825,
      "rewards/margins": 0.023504754528403282,
      "rewards/rejected": -0.14941677451133728,
      "sft_loss": 1.2591201066970825,
      "step": 490
    },
    {
      "epoch": 0.808244089715094,
      "grad_norm": 1.509691596031189,
      "learning_rate": 4.155353766456497e-06,
      "logits/chosen": -22.305217742919922,
      "logits/rejected": -22.103668212890625,
      "logps/chosen": -1.335773229598999,
      "logps/rejected": -1.4481773376464844,
      "loss": 1.4057,
      "odds_ratio_loss": 0.6996302604675293,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13357731699943542,
      "rewards/margins": 0.011240402236580849,
      "rewards/rejected": -0.14481773972511292,
      "sft_loss": 1.335773229598999,
      "step": 500
    },
    {
      "epoch": 0.808244089715094,
      "eval_logits/chosen": -21.914384841918945,
      "eval_logits/rejected": -21.94023323059082,
      "eval_logps/chosen": -1.31483793258667,
      "eval_logps/rejected": -1.4522359371185303,
      "eval_loss": 1.38706636428833,
      "eval_odds_ratio_loss": 0.7222831845283508,
      "eval_rewards/accuracies": 0.5318182110786438,
      "eval_rewards/chosen": -0.131483793258667,
      "eval_rewards/margins": 0.01373977866023779,
      "eval_rewards/rejected": -0.14522357285022736,
      "eval_runtime": 82.3592,
      "eval_samples_per_second": 13.356,
      "eval_sft_loss": 1.31483793258667,
      "eval_steps_per_second": 6.678,
      "step": 500
    },
    {
      "epoch": 0.8244089715093958,
      "grad_norm": 1.657311201095581,
      "learning_rate": 4.123370445773134e-06,
      "logits/chosen": -22.07869529724121,
      "logits/rejected": -22.091659545898438,
      "logps/chosen": -1.2787725925445557,
      "logps/rejected": -1.3090568780899048,
      "loss": 1.3519,
      "odds_ratio_loss": 0.731525719165802,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12787726521492004,
      "rewards/margins": 0.003028427716344595,
      "rewards/rejected": -0.13090567290782928,
      "sft_loss": 1.2787725925445557,
      "step": 510
    },
    {
      "epoch": 0.8405738533036977,
      "grad_norm": 4.675573825836182,
      "learning_rate": 4.090920965761906e-06,
      "logits/chosen": -21.979219436645508,
      "logits/rejected": -21.770206451416016,
      "logps/chosen": -1.304009199142456,
      "logps/rejected": -1.4551854133605957,
      "loss": 1.3752,
      "odds_ratio_loss": 0.7114746570587158,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13040092587471008,
      "rewards/margins": 0.015117612667381763,
      "rewards/rejected": -0.14551854133605957,
      "sft_loss": 1.304009199142456,
      "step": 520
    },
    {
      "epoch": 0.8567387350979996,
      "grad_norm": 4.995480060577393,
      "learning_rate": 4.058014644460991e-06,
      "logits/chosen": -21.77149772644043,
      "logits/rejected": -21.79526710510254,
      "logps/chosen": -1.3110116720199585,
      "logps/rejected": -1.4032762050628662,
      "loss": 1.3819,
      "odds_ratio_loss": 0.708578884601593,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13110119104385376,
      "rewards/margins": 0.009226455353200436,
      "rewards/rejected": -0.14032763242721558,
      "sft_loss": 1.3110116720199585,
      "step": 530
    },
    {
      "epoch": 0.8729036168923014,
      "grad_norm": 1.891120433807373,
      "learning_rate": 4.024660931092939e-06,
      "logits/chosen": -21.345439910888672,
      "logits/rejected": -21.40594482421875,
      "logps/chosen": -1.3298126459121704,
      "logps/rejected": -1.4730676412582397,
      "loss": 1.4012,
      "odds_ratio_loss": 0.713805079460144,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13298127055168152,
      "rewards/margins": 0.014325493946671486,
      "rewards/rejected": -0.14730675518512726,
      "sft_loss": 1.3298126459121704,
      "step": 540
    },
    {
      "epoch": 0.8890684986866033,
      "grad_norm": 7.624050617218018,
      "learning_rate": 3.990869403351272e-06,
      "logits/chosen": -21.680253982543945,
      "logits/rejected": -21.578588485717773,
      "logps/chosen": -1.3242005109786987,
      "logps/rejected": -1.496756911277771,
      "loss": 1.3887,
      "odds_ratio_loss": 0.644872784614563,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13242004811763763,
      "rewards/margins": 0.017255645245313644,
      "rewards/rejected": -0.14967571198940277,
      "sft_loss": 1.3242005109786987,
      "step": 550
    },
    {
      "epoch": 0.9052333804809052,
      "grad_norm": 2.1821932792663574,
      "learning_rate": 3.956649764650206e-06,
      "logits/chosen": -22.087711334228516,
      "logits/rejected": -22.087757110595703,
      "logps/chosen": -1.2911579608917236,
      "logps/rejected": -1.4688221216201782,
      "loss": 1.3633,
      "odds_ratio_loss": 0.7218787670135498,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.12911580502986908,
      "rewards/margins": 0.017766421660780907,
      "rewards/rejected": -0.14688222110271454,
      "sft_loss": 1.2911579608917236,
      "step": 560
    },
    {
      "epoch": 0.9213982622752072,
      "grad_norm": 5.107442855834961,
      "learning_rate": 3.92201184133826e-06,
      "logits/chosen": -21.773061752319336,
      "logits/rejected": -21.753795623779297,
      "logps/chosen": -1.2945088148117065,
      "logps/rejected": -1.4896302223205566,
      "loss": 1.3613,
      "odds_ratio_loss": 0.6681925654411316,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12945088744163513,
      "rewards/margins": 0.019512150436639786,
      "rewards/rejected": -0.14896303415298462,
      "sft_loss": 1.2945088148117065,
      "step": 570
    },
    {
      "epoch": 0.937563144069509,
      "grad_norm": 1.4870102405548096,
      "learning_rate": 3.886965579876572e-06,
      "logits/chosen": -21.675262451171875,
      "logits/rejected": -21.543224334716797,
      "logps/chosen": -1.3047420978546143,
      "logps/rejected": -1.3921304941177368,
      "loss": 1.3741,
      "odds_ratio_loss": 0.694009006023407,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1304742395877838,
      "rewards/margins": 0.008738822303712368,
      "rewards/rejected": -0.13921304047107697,
      "sft_loss": 1.3047420978546143,
      "step": 580
    },
    {
      "epoch": 0.9537280258638109,
      "grad_norm": 2.04402494430542,
      "learning_rate": 3.851521043982716e-06,
      "logits/chosen": -21.711593627929688,
      "logits/rejected": -21.78032875061035,
      "logps/chosen": -1.311596155166626,
      "logps/rejected": -1.351053237915039,
      "loss": 1.3862,
      "odds_ratio_loss": 0.7462160587310791,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.13115961849689484,
      "rewards/margins": 0.00394571665674448,
      "rewards/rejected": -0.13510534167289734,
      "sft_loss": 1.311596155166626,
      "step": 590
    },
    {
      "epoch": 0.9698929076581128,
      "grad_norm": 8.317533493041992,
      "learning_rate": 3.81568841174086e-06,
      "logits/chosen": -21.358070373535156,
      "logits/rejected": -21.403278350830078,
      "logps/chosen": -1.361669898033142,
      "logps/rejected": -1.494240403175354,
      "loss": 1.4338,
      "odds_ratio_loss": 0.7209025025367737,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1361669898033142,
      "rewards/margins": 0.013257039710879326,
      "rewards/rejected": -0.14942404627799988,
      "sft_loss": 1.361669898033142,
      "step": 600
    },
    {
      "epoch": 0.9860577894524146,
      "grad_norm": 2.044271469116211,
      "learning_rate": 3.7794779726790664e-06,
      "logits/chosen": -21.353389739990234,
      "logits/rejected": -21.419639587402344,
      "logps/chosen": -1.222917914390564,
      "logps/rejected": -1.3896757364273071,
      "loss": 1.2907,
      "odds_ratio_loss": 0.6782225370407104,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12229180335998535,
      "rewards/margins": 0.016675781458616257,
      "rewards/rejected": -0.1389675885438919,
      "sft_loss": 1.222917914390564,
      "step": 610
    },
    {
      "epoch": 1.0022226712467166,
      "grad_norm": 4.229960918426514,
      "learning_rate": 3.7429001248146096e-06,
      "logits/chosen": -21.391468048095703,
      "logits/rejected": -21.433670043945312,
      "logps/chosen": -1.3207805156707764,
      "logps/rejected": -1.5086777210235596,
      "loss": 1.3864,
      "odds_ratio_loss": 0.655738115310669,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13207808136940002,
      "rewards/margins": 0.018789704889059067,
      "rewards/rejected": -0.1508677750825882,
      "sft_loss": 1.3207805156707764,
      "step": 620
    },
    {
      "epoch": 1.0183875530410185,
      "grad_norm": 1.3560584783554077,
      "learning_rate": 3.7059653716681227e-06,
      "logits/chosen": -21.297557830810547,
      "logits/rejected": -21.24813461303711,
      "logps/chosen": -1.3175418376922607,
      "logps/rejected": -1.4784363508224487,
      "loss": 1.3899,
      "odds_ratio_loss": 0.7236071825027466,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13175417482852936,
      "rewards/margins": 0.016089454293251038,
      "rewards/rejected": -0.1478436291217804,
      "sft_loss": 1.3175418376922607,
      "step": 630
    },
    {
      "epoch": 1.0345524348353203,
      "grad_norm": 4.676194190979004,
      "learning_rate": 3.668684319247463e-06,
      "logits/chosen": -21.50934410095215,
      "logits/rejected": -21.4769229888916,
      "logps/chosen": -1.2353088855743408,
      "logps/rejected": -1.5056722164154053,
      "loss": 1.298,
      "odds_ratio_loss": 0.6273224949836731,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12353090196847916,
      "rewards/margins": 0.02703634463250637,
      "rewards/rejected": -0.15056723356246948,
      "sft_loss": 1.2353088855743408,
      "step": 640
    },
    {
      "epoch": 1.0507173166296222,
      "grad_norm": 1.456129550933838,
      "learning_rate": 3.6310676730021373e-06,
      "logits/chosen": -21.462495803833008,
      "logits/rejected": -21.57852554321289,
      "logps/chosen": -1.2601442337036133,
      "logps/rejected": -1.3218932151794434,
      "loss": 1.3324,
      "odds_ratio_loss": 0.7224392890930176,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12601442635059357,
      "rewards/margins": 0.006174898706376553,
      "rewards/rejected": -0.1321893036365509,
      "sft_loss": 1.2601442337036133,
      "step": 650
    },
    {
      "epoch": 1.066882198423924,
      "grad_norm": 3.662935733795166,
      "learning_rate": 3.593126234749178e-06,
      "logits/chosen": -21.538293838500977,
      "logits/rejected": -21.53152847290039,
      "logps/chosen": -1.3035653829574585,
      "logps/rejected": -1.4283910989761353,
      "loss": 1.375,
      "odds_ratio_loss": 0.7146205902099609,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1303565502166748,
      "rewards/margins": 0.012482564896345139,
      "rewards/rejected": -0.14283910393714905,
      "sft_loss": 1.3035653829574585,
      "step": 660
    },
    {
      "epoch": 1.083047080218226,
      "grad_norm": 2.5778450965881348,
      "learning_rate": 3.554870899571343e-06,
      "logits/chosen": -21.25736427307129,
      "logits/rejected": -21.203954696655273,
      "logps/chosen": -1.2814052104949951,
      "logps/rejected": -1.4217122793197632,
      "loss": 1.3498,
      "odds_ratio_loss": 0.6842816472053528,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12814053893089294,
      "rewards/margins": 0.0140307005494833,
      "rewards/rejected": -0.1421712338924408,
      "sft_loss": 1.2814052104949951,
      "step": 670
    },
    {
      "epoch": 1.0992119620125278,
      "grad_norm": 3.7894856929779053,
      "learning_rate": 3.5163126526885373e-06,
      "logits/chosen": -21.061431884765625,
      "logits/rejected": -21.307607650756836,
      "logps/chosen": -1.2376093864440918,
      "logps/rejected": -1.3728429079055786,
      "loss": 1.3095,
      "odds_ratio_loss": 0.7185710668563843,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12376093864440918,
      "rewards/margins": 0.013523365370929241,
      "rewards/rejected": -0.1372843086719513,
      "sft_loss": 1.2376093864440918,
      "step": 680
    },
    {
      "epoch": 1.1153768438068297,
      "grad_norm": 2.3537509441375732,
      "learning_rate": 3.4774625663033484e-06,
      "logits/chosen": -21.21095085144043,
      "logits/rejected": -21.2464542388916,
      "logps/chosen": -1.232604742050171,
      "logps/rejected": -1.3756181001663208,
      "loss": 1.3002,
      "odds_ratio_loss": 0.6756417155265808,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.1232604831457138,
      "rewards/margins": 0.014301331713795662,
      "rewards/rejected": -0.1375618278980255,
      "sft_loss": 1.232604742050171,
      "step": 690
    },
    {
      "epoch": 1.1315417256011315,
      "grad_norm": 1.4719160795211792,
      "learning_rate": 3.4383317964216067e-06,
      "logits/chosen": -21.036582946777344,
      "logits/rejected": -21.25110626220703,
      "logps/chosen": -1.2365304231643677,
      "logps/rejected": -1.3071203231811523,
      "loss": 1.3116,
      "odds_ratio_loss": 0.7511636018753052,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.12365303188562393,
      "rewards/margins": 0.007059005089104176,
      "rewards/rejected": -0.13071204721927643,
      "sft_loss": 1.2365304231643677,
      "step": 700
    },
    {
      "epoch": 1.1477066073954334,
      "grad_norm": 2.8253660202026367,
      "learning_rate": 3.398931579648877e-06,
      "logits/chosen": -21.2774658203125,
      "logits/rejected": -21.474933624267578,
      "logps/chosen": -1.3233165740966797,
      "logps/rejected": -1.49609375,
      "loss": 1.3932,
      "odds_ratio_loss": 0.6990328431129456,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13233166933059692,
      "rewards/margins": 0.017277712002396584,
      "rewards/rejected": -0.14960938692092896,
      "sft_loss": 1.3233165740966797,
      "step": 710
    },
    {
      "epoch": 1.1638714891897353,
      "grad_norm": 2.0079872608184814,
      "learning_rate": 3.359273229963813e-06,
      "logits/chosen": -21.300012588500977,
      "logits/rejected": -21.20311737060547,
      "logps/chosen": -1.2444874048233032,
      "logps/rejected": -1.3516159057617188,
      "loss": 1.315,
      "odds_ratio_loss": 0.7051676511764526,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1244487538933754,
      "rewards/margins": 0.010712863877415657,
      "rewards/rejected": -0.1351616084575653,
      "sft_loss": 1.2444874048233032,
      "step": 720
    },
    {
      "epoch": 1.1800363709840371,
      "grad_norm": 1.6793773174285889,
      "learning_rate": 3.319368135469285e-06,
      "logits/chosen": -21.23295021057129,
      "logits/rejected": -21.303585052490234,
      "logps/chosen": -1.294540524482727,
      "logps/rejected": -1.422060251235962,
      "loss": 1.3682,
      "odds_ratio_loss": 0.7366825938224792,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12945404648780823,
      "rewards/margins": 0.012751981616020203,
      "rewards/rejected": -0.14220602810382843,
      "sft_loss": 1.294540524482727,
      "step": 730
    },
    {
      "epoch": 1.196201252778339,
      "grad_norm": 2.9670250415802,
      "learning_rate": 3.279227755122228e-06,
      "logits/chosen": -21.109752655029297,
      "logits/rejected": -21.158111572265625,
      "logps/chosen": -1.2494882345199585,
      "logps/rejected": -1.5186522006988525,
      "loss": 1.3163,
      "odds_ratio_loss": 0.6685449481010437,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12494882196187973,
      "rewards/margins": 0.02691638469696045,
      "rewards/rejected": -0.15186521410942078,
      "sft_loss": 1.2494882345199585,
      "step": 740
    },
    {
      "epoch": 1.2123661345726409,
      "grad_norm": 2.090475082397461,
      "learning_rate": 3.2388636154431417e-06,
      "logits/chosen": -21.39287757873535,
      "logits/rejected": -21.400285720825195,
      "logps/chosen": -1.3509362936019897,
      "logps/rejected": -1.5079971551895142,
      "loss": 1.4216,
      "odds_ratio_loss": 0.7062050104141235,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13509364426136017,
      "rewards/margins": 0.015706082805991173,
      "rewards/rejected": -0.1507997065782547,
      "sft_loss": 1.3509362936019897,
      "step": 750
    },
    {
      "epoch": 1.2285310163669427,
      "grad_norm": 1.8322948217391968,
      "learning_rate": 3.198287307206192e-06,
      "logits/chosen": -20.998327255249023,
      "logits/rejected": -21.087013244628906,
      "logps/chosen": -1.254172921180725,
      "logps/rejected": -1.3801274299621582,
      "loss": 1.3241,
      "odds_ratio_loss": 0.6988333463668823,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1254173070192337,
      "rewards/margins": 0.012595455162227154,
      "rewards/rejected": -0.13801275193691254,
      "sft_loss": 1.254172921180725,
      "step": 760
    },
    {
      "epoch": 1.2446958981612446,
      "grad_norm": 1.5089768171310425,
      "learning_rate": 3.157510482110856e-06,
      "logits/chosen": -21.451335906982422,
      "logits/rejected": -21.22077751159668,
      "logps/chosen": -1.2650538682937622,
      "logps/rejected": -1.3639296293258667,
      "loss": 1.337,
      "odds_ratio_loss": 0.7199094891548157,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12650538980960846,
      "rewards/margins": 0.009887583553791046,
      "rewards/rejected": -0.1363929808139801,
      "sft_loss": 1.2650538682937622,
      "step": 770
    },
    {
      "epoch": 1.2608607799555465,
      "grad_norm": 1.1987851858139038,
      "learning_rate": 3.116544849436077e-06,
      "logits/chosen": -21.068490982055664,
      "logits/rejected": -20.816608428955078,
      "logps/chosen": -1.3564980030059814,
      "logps/rejected": -1.5898200273513794,
      "loss": 1.4243,
      "odds_ratio_loss": 0.6784577369689941,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13564978539943695,
      "rewards/margins": 0.023332208395004272,
      "rewards/rejected": -0.15898200869560242,
      "sft_loss": 1.3564980030059814,
      "step": 780
    },
    {
      "epoch": 1.2770256617498483,
      "grad_norm": 1.9816875457763672,
      "learning_rate": 3.0754021726778848e-06,
      "logits/chosen": -21.14816665649414,
      "logits/rejected": -21.171310424804688,
      "logps/chosen": -1.1857916116714478,
      "logps/rejected": -1.4323453903198242,
      "loss": 1.2508,
      "odds_ratio_loss": 0.6496400833129883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11857914924621582,
      "rewards/margins": 0.02465539053082466,
      "rewards/rejected": -0.14323453605175018,
      "sft_loss": 1.1857916116714478,
      "step": 790
    },
    {
      "epoch": 1.2931905435441502,
      "grad_norm": 0.9774518609046936,
      "learning_rate": 3.0340942661714463e-06,
      "logits/chosen": -21.33401870727539,
      "logits/rejected": -21.100515365600586,
      "logps/chosen": -1.3348338603973389,
      "logps/rejected": -1.4366466999053955,
      "loss": 1.4051,
      "odds_ratio_loss": 0.7027276754379272,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1334833949804306,
      "rewards/margins": 0.010181264951825142,
      "rewards/rejected": -0.1436646580696106,
      "sft_loss": 1.3348338603973389,
      "step": 800
    },
    {
      "epoch": 1.3093554253384523,
      "grad_norm": 2.8201749324798584,
      "learning_rate": 2.992632991698512e-06,
      "logits/chosen": -20.885223388671875,
      "logits/rejected": -21.0069522857666,
      "logps/chosen": -1.28475022315979,
      "logps/rejected": -1.434466004371643,
      "loss": 1.3541,
      "odds_ratio_loss": 0.6936103105545044,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12847502529621124,
      "rewards/margins": 0.014971581287682056,
      "rewards/rejected": -0.14344659447669983,
      "sft_loss": 1.28475022315979,
      "step": 810
    },
    {
      "epoch": 1.3255203071327541,
      "grad_norm": 1.557600498199463,
      "learning_rate": 2.9510302550812537e-06,
      "logits/chosen": -21.287960052490234,
      "logits/rejected": -21.395370483398438,
      "logps/chosen": -1.2278238534927368,
      "logps/rejected": -1.4308351278305054,
      "loss": 1.2925,
      "odds_ratio_loss": 0.646266520023346,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1227823942899704,
      "rewards/margins": 0.02030111663043499,
      "rewards/rejected": -0.14308352768421173,
      "sft_loss": 1.2278238534927368,
      "step": 820
    },
    {
      "epoch": 1.341685188927056,
      "grad_norm": 2.6895642280578613,
      "learning_rate": 2.9092980027634325e-06,
      "logits/chosen": -21.14548110961914,
      "logits/rejected": -21.261184692382812,
      "logps/chosen": -1.1719629764556885,
      "logps/rejected": -1.3467178344726562,
      "loss": 1.2381,
      "odds_ratio_loss": 0.6613830327987671,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11719629913568497,
      "rewards/margins": 0.017475496977567673,
      "rewards/rejected": -0.13467177748680115,
      "sft_loss": 1.1719629764556885,
      "step": 830
    },
    {
      "epoch": 1.3578500707213579,
      "grad_norm": 1.8690439462661743,
      "learning_rate": 2.867448218379927e-06,
      "logits/chosen": -21.13509750366211,
      "logits/rejected": -21.14713478088379,
      "logps/chosen": -1.2881317138671875,
      "logps/rejected": -1.419306993484497,
      "loss": 1.359,
      "odds_ratio_loss": 0.7090221643447876,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12881317734718323,
      "rewards/margins": 0.013117517344653606,
      "rewards/rejected": -0.1419306844472885,
      "sft_loss": 1.2881317138671875,
      "step": 840
    },
    {
      "epoch": 1.3740149525156597,
      "grad_norm": 4.924416542053223,
      "learning_rate": 2.825492919315559e-06,
      "logits/chosen": -20.981931686401367,
      "logits/rejected": -21.142297744750977,
      "logps/chosen": -1.3466829061508179,
      "logps/rejected": -1.3898065090179443,
      "loss": 1.4234,
      "odds_ratio_loss": 0.7671664953231812,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1346682757139206,
      "rewards/margins": 0.004312378820031881,
      "rewards/rejected": -0.1389806568622589,
      "sft_loss": 1.3466829061508179,
      "step": 850
    },
    {
      "epoch": 1.3901798343099616,
      "grad_norm": 1.9793908596038818,
      "learning_rate": 2.7834441532542482e-06,
      "logits/chosen": -21.344064712524414,
      "logits/rejected": -21.355037689208984,
      "logps/chosen": -1.2326418161392212,
      "logps/rejected": -1.3959559202194214,
      "loss": 1.3028,
      "odds_ratio_loss": 0.7015843987464905,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12326417118310928,
      "rewards/margins": 0.016331423074007034,
      "rewards/rejected": -0.13959559798240662,
      "sft_loss": 1.2326418161392212,
      "step": 860
    },
    {
      "epoch": 1.4063447161042635,
      "grad_norm": 1.5754936933517456,
      "learning_rate": 2.74131399471945e-06,
      "logits/chosen": -21.10004234313965,
      "logits/rejected": -21.218639373779297,
      "logps/chosen": -1.2864024639129639,
      "logps/rejected": -1.3692989349365234,
      "loss": 1.3586,
      "odds_ratio_loss": 0.7215597033500671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12864024937152863,
      "rewards/margins": 0.008289644494652748,
      "rewards/rejected": -0.13692989945411682,
      "sft_loss": 1.2864024639129639,
      "step": 870
    },
    {
      "epoch": 1.4225095978985653,
      "grad_norm": 2.591798782348633,
      "learning_rate": 2.6991145416068947e-06,
      "logits/chosen": -20.910709381103516,
      "logits/rejected": -21.049091339111328,
      "logps/chosen": -1.2924576997756958,
      "logps/rejected": -1.3584909439086914,
      "loss": 1.3658,
      "odds_ratio_loss": 0.7329753637313843,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12924575805664062,
      "rewards/margins": 0.006603321526199579,
      "rewards/rejected": -0.13584908843040466,
      "sft_loss": 1.2924576997756958,
      "step": 880
    },
    {
      "epoch": 1.4386744796928672,
      "grad_norm": 1.6928390264511108,
      "learning_rate": 2.6568579117106143e-06,
      "logits/chosen": -20.872098922729492,
      "logits/rejected": -20.832103729248047,
      "logps/chosen": -1.2083466053009033,
      "logps/rejected": -1.406185507774353,
      "loss": 1.2749,
      "odds_ratio_loss": 0.6651225090026855,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12083464860916138,
      "rewards/margins": 0.019783901050686836,
      "rewards/rejected": -0.14061854779720306,
      "sft_loss": 1.2083466053009033,
      "step": 890
    },
    {
      "epoch": 1.454839361487169,
      "grad_norm": 0.9949257373809814,
      "learning_rate": 2.6145562392432544e-06,
      "logits/chosen": -20.952428817749023,
      "logits/rejected": -20.857725143432617,
      "logps/chosen": -1.2628931999206543,
      "logps/rejected": -1.3288319110870361,
      "loss": 1.3382,
      "odds_ratio_loss": 0.75352543592453,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12628932297229767,
      "rewards/margins": 0.006593865808099508,
      "rewards/rejected": -0.1328831911087036,
      "sft_loss": 1.2628931999206543,
      "step": 900
    },
    {
      "epoch": 1.471004243281471,
      "grad_norm": 2.0959157943725586,
      "learning_rate": 2.5722216713516682e-06,
      "logits/chosen": -20.460887908935547,
      "logits/rejected": -20.67595100402832,
      "logps/chosen": -1.2154316902160645,
      "logps/rejected": -1.3711063861846924,
      "loss": 1.284,
      "odds_ratio_loss": 0.6853248476982117,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12154315412044525,
      "rewards/margins": 0.015567478723824024,
      "rewards/rejected": -0.1371106207370758,
      "sft_loss": 1.2154316902160645,
      "step": 910
    },
    {
      "epoch": 1.4871691250757728,
      "grad_norm": 2.9631924629211426,
      "learning_rate": 2.5298663646288064e-06,
      "logits/chosen": -20.95013999938965,
      "logits/rejected": -21.05425453186035,
      "logps/chosen": -1.213805079460144,
      "logps/rejected": -1.382340669631958,
      "loss": 1.2822,
      "odds_ratio_loss": 0.6836367845535278,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1213805228471756,
      "rewards/margins": 0.01685354672372341,
      "rewards/rejected": -0.13823407888412476,
      "sft_loss": 1.213805079460144,
      "step": 920
    },
    {
      "epoch": 1.503334006870075,
      "grad_norm": 2.637305498123169,
      "learning_rate": 2.487502481622879e-06,
      "logits/chosen": -20.81560516357422,
      "logits/rejected": -20.701404571533203,
      "logps/chosen": -1.3175880908966064,
      "logps/rejected": -1.4211034774780273,
      "loss": 1.3892,
      "odds_ratio_loss": 0.7159314155578613,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13175880908966064,
      "rewards/margins": 0.010351541452109814,
      "rewards/rejected": -0.14211036264896393,
      "sft_loss": 1.3175880908966064,
      "step": 930
    },
    {
      "epoch": 1.5194988886643768,
      "grad_norm": 1.9374908208847046,
      "learning_rate": 2.4451421873448253e-06,
      "logits/chosen": -20.670528411865234,
      "logits/rejected": -20.75124168395996,
      "logps/chosen": -1.2572494745254517,
      "logps/rejected": -1.367071509361267,
      "loss": 1.3282,
      "odds_ratio_loss": 0.7093643546104431,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12572495639324188,
      "rewards/margins": 0.010982205159962177,
      "rewards/rejected": -0.13670717179775238,
      "sft_loss": 1.2572494745254517,
      "step": 940
    },
    {
      "epoch": 1.5356637704586786,
      "grad_norm": 3.949268102645874,
      "learning_rate": 2.40279764577506e-06,
      "logits/chosen": -21.262102127075195,
      "logits/rejected": -21.309324264526367,
      "logps/chosen": -1.3575810194015503,
      "logps/rejected": -1.4113634824752808,
      "loss": 1.4328,
      "odds_ratio_loss": 0.7524511218070984,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.13575811684131622,
      "rewards/margins": 0.005378241650760174,
      "rewards/rejected": -0.14113636314868927,
      "sft_loss": 1.3575810194015503,
      "step": 950
    },
    {
      "epoch": 1.5518286522529805,
      "grad_norm": 1.4890868663787842,
      "learning_rate": 2.3604810163705242e-06,
      "logits/chosen": -20.816661834716797,
      "logits/rejected": -20.944917678833008,
      "logps/chosen": -1.2137186527252197,
      "logps/rejected": -1.348968505859375,
      "loss": 1.2808,
      "odds_ratio_loss": 0.6708163022994995,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12137186527252197,
      "rewards/margins": 0.013524973765015602,
      "rewards/rejected": -0.13489684462547302,
      "sft_loss": 1.2137186527252197,
      "step": 960
    },
    {
      "epoch": 1.5679935340472824,
      "grad_norm": 2.5744829177856445,
      "learning_rate": 2.3182044505730364e-06,
      "logits/chosen": -20.806903839111328,
      "logits/rejected": -20.799480438232422,
      "logps/chosen": -1.170987606048584,
      "logps/rejected": -1.3193377256393433,
      "loss": 1.2392,
      "odds_ratio_loss": 0.6825143098831177,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.11709876358509064,
      "rewards/margins": 0.014834997244179249,
      "rewards/rejected": -0.1319337785243988,
      "sft_loss": 1.170987606048584,
      "step": 970
    },
    {
      "epoch": 1.5841584158415842,
      "grad_norm": 3.971120834350586,
      "learning_rate": 2.275980088319941e-06,
      "logits/chosen": -21.002077102661133,
      "logits/rejected": -20.75215721130371,
      "logps/chosen": -1.2036049365997314,
      "logps/rejected": -1.2877798080444336,
      "loss": 1.2758,
      "odds_ratio_loss": 0.7220691442489624,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12036049365997314,
      "rewards/margins": 0.008417483419179916,
      "rewards/rejected": -0.12877798080444336,
      "sft_loss": 1.2036049365997314,
      "step": 980
    },
    {
      "epoch": 1.600323297635886,
      "grad_norm": 2.0432345867156982,
      "learning_rate": 2.2338200545580577e-06,
      "logits/chosen": -20.597959518432617,
      "logits/rejected": -20.82900619506836,
      "logps/chosen": -1.182715654373169,
      "logps/rejected": -1.4004007577896118,
      "loss": 1.2503,
      "odds_ratio_loss": 0.6758050322532654,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.11827157437801361,
      "rewards/margins": 0.021768497303128242,
      "rewards/rejected": -0.1400400698184967,
      "sft_loss": 1.182715654373169,
      "step": 990
    },
    {
      "epoch": 1.616488179430188,
      "grad_norm": 2.0639455318450928,
      "learning_rate": 2.191736455761947e-06,
      "logits/chosen": -20.99373435974121,
      "logits/rejected": -20.91073989868164,
      "logps/chosen": -1.153194785118103,
      "logps/rejected": -1.2575992345809937,
      "loss": 1.2196,
      "odds_ratio_loss": 0.663902223110199,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.11531949043273926,
      "rewards/margins": 0.010440433397889137,
      "rewards/rejected": -0.12575991451740265,
      "sft_loss": 1.153194785118103,
      "step": 1000
    },
    {
      "epoch": 1.616488179430188,
      "eval_logits/chosen": -20.882339477539062,
      "eval_logits/rejected": -20.911054611206055,
      "eval_logps/chosen": -1.2678194046020508,
      "eval_logps/rejected": -1.4078764915466309,
      "eval_loss": 1.3396929502487183,
      "eval_odds_ratio_loss": 0.7187345623970032,
      "eval_rewards/accuracies": 0.5354545712471008,
      "eval_rewards/chosen": -0.12678195536136627,
      "eval_rewards/margins": 0.01400570198893547,
      "eval_rewards/rejected": -0.14078764617443085,
      "eval_runtime": 158.823,
      "eval_samples_per_second": 6.926,
      "eval_sft_loss": 1.2678194046020508,
      "eval_steps_per_second": 3.463,
      "step": 1000
    },
    {
      "epoch": 1.6326530612244898,
      "grad_norm": 1.447840929031372,
      "learning_rate": 2.1497413764574673e-06,
      "logits/chosen": -21.098125457763672,
      "logits/rejected": -21.02555274963379,
      "logps/chosen": -1.2790714502334595,
      "logps/rejected": -1.4720830917358398,
      "loss": 1.3456,
      "odds_ratio_loss": 0.6656414270401001,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1279071420431137,
      "rewards/margins": 0.01930117979645729,
      "rewards/rejected": -0.1472083032131195,
      "sft_loss": 1.2790714502334595,
      "step": 1010
    },
    {
      "epoch": 1.6488179430187917,
      "grad_norm": 2.2929062843322754,
      "learning_rate": 2.1078468757516395e-06,
      "logits/chosen": -21.029102325439453,
      "logits/rejected": -20.96735382080078,
      "logps/chosen": -1.1978142261505127,
      "logps/rejected": -1.2682490348815918,
      "loss": 1.2676,
      "odds_ratio_loss": 0.6979144811630249,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.11978141963481903,
      "rewards/margins": 0.007043488323688507,
      "rewards/rejected": -0.12682488560676575,
      "sft_loss": 1.1978142261505127,
      "step": 1020
    },
    {
      "epoch": 1.6649828248130936,
      "grad_norm": 1.679506778717041,
      "learning_rate": 2.0660649838698145e-06,
      "logits/chosen": -21.126895904541016,
      "logits/rejected": -21.121044158935547,
      "logps/chosen": -1.2263692617416382,
      "logps/rejected": -1.3581560850143433,
      "loss": 1.2969,
      "odds_ratio_loss": 0.7051782011985779,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12263691425323486,
      "rewards/margins": 0.013178691267967224,
      "rewards/rejected": -0.1358156055212021,
      "sft_loss": 1.2263692617416382,
      "step": 1030
    },
    {
      "epoch": 1.6811477066073954,
      "grad_norm": 1.5033296346664429,
      "learning_rate": 2.0244076987011284e-06,
      "logits/chosen": -21.097610473632812,
      "logits/rejected": -21.023983001708984,
      "logps/chosen": -1.263479232788086,
      "logps/rejected": -1.430101990699768,
      "loss": 1.3286,
      "odds_ratio_loss": 0.6507799029350281,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12634792923927307,
      "rewards/margins": 0.01666227914392948,
      "rewards/rejected": -0.1430101990699768,
      "sft_loss": 1.263479232788086,
      "step": 1040
    },
    {
      "epoch": 1.6973125884016973,
      "grad_norm": 1.782820463180542,
      "learning_rate": 1.982886982353251e-06,
      "logits/chosen": -20.983699798583984,
      "logits/rejected": -20.792827606201172,
      "logps/chosen": -1.2545843124389648,
      "logps/rejected": -1.4638617038726807,
      "loss": 1.3236,
      "odds_ratio_loss": 0.6896650195121765,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12545843422412872,
      "rewards/margins": 0.020927732810378075,
      "rewards/rejected": -0.14638619124889374,
      "sft_loss": 1.2545843124389648,
      "step": 1050
    },
    {
      "epoch": 1.7134774701959992,
      "grad_norm": 2.0116512775421143,
      "learning_rate": 1.941514757717392e-06,
      "logits/chosen": -20.63968276977539,
      "logits/rejected": -20.679515838623047,
      "logps/chosen": -1.2405683994293213,
      "logps/rejected": -1.4277904033660889,
      "loss": 1.3065,
      "odds_ratio_loss": 0.6590927839279175,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12405683845281601,
      "rewards/margins": 0.018722202628850937,
      "rewards/rejected": -0.14277902245521545,
      "sft_loss": 1.2405683994293213,
      "step": 1060
    },
    {
      "epoch": 1.729642351990301,
      "grad_norm": 2.4715521335601807,
      "learning_rate": 1.9003029050445953e-06,
      "logits/chosen": -20.77858543395996,
      "logits/rejected": -20.901081085205078,
      "logps/chosen": -1.2767174243927002,
      "logps/rejected": -1.3766672611236572,
      "loss": 1.3478,
      "odds_ratio_loss": 0.7110922336578369,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1276717483997345,
      "rewards/margins": 0.00999497715383768,
      "rewards/rejected": -0.137666717171669,
      "sft_loss": 1.2767174243927002,
      "step": 1070
    },
    {
      "epoch": 1.745807233784603,
      "grad_norm": 1.7757654190063477,
      "learning_rate": 1.8592632585342523e-06,
      "logits/chosen": -20.718509674072266,
      "logits/rejected": -20.882915496826172,
      "logps/chosen": -1.2459039688110352,
      "logps/rejected": -1.3744847774505615,
      "loss": 1.3167,
      "odds_ratio_loss": 0.7076380848884583,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12459039688110352,
      "rewards/margins": 0.012858082540333271,
      "rewards/rejected": -0.13744845986366272,
      "sft_loss": 1.2459039688110352,
      "step": 1080
    },
    {
      "epoch": 1.7619721155789048,
      "grad_norm": 8.900938034057617,
      "learning_rate": 1.8184076029358527e-06,
      "logits/chosen": -20.89524269104004,
      "logits/rejected": -20.46439552307129,
      "logps/chosen": -1.1931703090667725,
      "logps/rejected": -1.2244675159454346,
      "loss": 1.2657,
      "odds_ratio_loss": 0.725264310836792,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11931703239679337,
      "rewards/margins": 0.003129716496914625,
      "rewards/rejected": -0.12244673818349838,
      "sft_loss": 1.1931703090667725,
      "step": 1090
    },
    {
      "epoch": 1.7781369973732066,
      "grad_norm": 1.4426664113998413,
      "learning_rate": 1.7777476701649318e-06,
      "logits/chosen": -20.599040985107422,
      "logits/rejected": -20.622554779052734,
      "logps/chosen": -1.2419286966323853,
      "logps/rejected": -1.3642092943191528,
      "loss": 1.3125,
      "odds_ratio_loss": 0.7057370543479919,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12419287115335464,
      "rewards/margins": 0.012228056788444519,
      "rewards/rejected": -0.13642092049121857,
      "sft_loss": 1.2419286966323853,
      "step": 1100
    },
    {
      "epoch": 1.7943018791675085,
      "grad_norm": 2.3391785621643066,
      "learning_rate": 1.7372951359341925e-06,
      "logits/chosen": -20.814512252807617,
      "logits/rejected": -20.616971969604492,
      "logps/chosen": -1.183668851852417,
      "logps/rejected": -1.3042553663253784,
      "loss": 1.2526,
      "odds_ratio_loss": 0.6895106434822083,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.11836691200733185,
      "rewards/margins": 0.012058648280799389,
      "rewards/rejected": -0.13042554259300232,
      "sft_loss": 1.183668851852417,
      "step": 1110
    },
    {
      "epoch": 1.8104667609618104,
      "grad_norm": 3.773312568664551,
      "learning_rate": 1.6970616164007547e-06,
      "logits/chosen": -20.64287567138672,
      "logits/rejected": -20.485708236694336,
      "logps/chosen": -1.1766704320907593,
      "logps/rejected": -1.3071858882904053,
      "loss": 1.2473,
      "odds_ratio_loss": 0.7061667442321777,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11766704171895981,
      "rewards/margins": 0.013051542453467846,
      "rewards/rejected": -0.13071858882904053,
      "sft_loss": 1.1766704320907593,
      "step": 1120
    },
    {
      "epoch": 1.8266316427561122,
      "grad_norm": 6.383511543273926,
      "learning_rate": 1.6570586648305276e-06,
      "logits/chosen": -20.542699813842773,
      "logits/rejected": -20.710830688476562,
      "logps/chosen": -1.2573511600494385,
      "logps/rejected": -1.453174114227295,
      "loss": 1.3268,
      "odds_ratio_loss": 0.6940013766288757,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1257351189851761,
      "rewards/margins": 0.019582299515604973,
      "rewards/rejected": -0.1453174203634262,
      "sft_loss": 1.2573511600494385,
      "step": 1130
    },
    {
      "epoch": 1.842796524550414,
      "grad_norm": 2.227006196975708,
      "learning_rate": 1.6172977682806151e-06,
      "logits/chosen": -20.67087173461914,
      "logits/rejected": -20.963016510009766,
      "logps/chosen": -1.254607915878296,
      "logps/rejected": -1.4263418912887573,
      "loss": 1.32,
      "odds_ratio_loss": 0.6536819338798523,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12546080350875854,
      "rewards/margins": 0.017173394560813904,
      "rewards/rejected": -0.14263419806957245,
      "sft_loss": 1.254607915878296,
      "step": 1140
    },
    {
      "epoch": 1.858961406344716,
      "grad_norm": 2.0494322776794434,
      "learning_rate": 1.5777903443007586e-06,
      "logits/chosen": -20.967281341552734,
      "logits/rejected": -20.45404624938965,
      "logps/chosen": -1.291761040687561,
      "logps/rejected": -1.4253360033035278,
      "loss": 1.3634,
      "odds_ratio_loss": 0.7167172431945801,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12917611002922058,
      "rewards/margins": 0.013357502408325672,
      "rewards/rejected": -0.14253361523151398,
      "sft_loss": 1.291761040687561,
      "step": 1150
    },
    {
      "epoch": 1.8751262881390178,
      "grad_norm": 2.5553345680236816,
      "learning_rate": 1.5385477376547226e-06,
      "logits/chosen": -20.862293243408203,
      "logits/rejected": -20.802623748779297,
      "logps/chosen": -1.3029327392578125,
      "logps/rejected": -1.4472944736480713,
      "loss": 1.3688,
      "odds_ratio_loss": 0.6585663557052612,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13029327988624573,
      "rewards/margins": 0.014436167664825916,
      "rewards/rejected": -0.14472945034503937,
      "sft_loss": 1.3029327392578125,
      "step": 1160
    },
    {
      "epoch": 1.89129116993332,
      "grad_norm": 1.7979766130447388,
      "learning_rate": 1.4995812170625845e-06,
      "logits/chosen": -20.667551040649414,
      "logits/rejected": -20.655712127685547,
      "logps/chosen": -1.317637324333191,
      "logps/rejected": -1.5292212963104248,
      "loss": 1.3849,
      "odds_ratio_loss": 0.6725908517837524,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1317637413740158,
      "rewards/margins": 0.021158380433917046,
      "rewards/rejected": -0.152922123670578,
      "sft_loss": 1.317637324333191,
      "step": 1170
    },
    {
      "epoch": 1.9074560517276218,
      "grad_norm": 7.151007652282715,
      "learning_rate": 1.4609019719648666e-06,
      "logits/chosen": -21.043447494506836,
      "logits/rejected": -20.968263626098633,
      "logps/chosen": -1.2828052043914795,
      "logps/rejected": -1.4710747003555298,
      "loss": 1.3489,
      "odds_ratio_loss": 0.6608654856681824,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12828053534030914,
      "rewards/margins": 0.018826942890882492,
      "rewards/rejected": -0.14710748195648193,
      "sft_loss": 1.2828052043914795,
      "step": 1180
    },
    {
      "epoch": 1.9236209335219236,
      "grad_norm": 3.7433314323425293,
      "learning_rate": 1.42252110930943e-06,
      "logits/chosen": -20.565446853637695,
      "logits/rejected": -20.41248893737793,
      "logps/chosen": -1.1272156238555908,
      "logps/rejected": -1.213004469871521,
      "loss": 1.2001,
      "odds_ratio_loss": 0.7286427021026611,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.11272156238555908,
      "rewards/margins": 0.008578887209296227,
      "rewards/rejected": -0.12130045890808105,
      "sft_loss": 1.1272156238555908,
      "step": 1190
    },
    {
      "epoch": 1.9397858153162255,
      "grad_norm": 2.740086793899536,
      "learning_rate": 1.3844496503620493e-06,
      "logits/chosen": -20.73773765563965,
      "logits/rejected": -20.874481201171875,
      "logps/chosen": -1.2707687616348267,
      "logps/rejected": -1.3804893493652344,
      "loss": 1.3383,
      "odds_ratio_loss": 0.6750306487083435,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1270768642425537,
      "rewards/margins": 0.010972070507705212,
      "rewards/rejected": -0.1380489468574524,
      "sft_loss": 1.2707687616348267,
      "step": 1200
    },
    {
      "epoch": 1.9559506971105274,
      "grad_norm": 3.2234959602355957,
      "learning_rate": 1.3466985275416081e-06,
      "logits/chosen": -20.732952117919922,
      "logits/rejected": -20.362863540649414,
      "logps/chosen": -1.3259879350662231,
      "logps/rejected": -1.436372995376587,
      "loss": 1.4005,
      "odds_ratio_loss": 0.7446285486221313,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13259878754615784,
      "rewards/margins": 0.011038513854146004,
      "rewards/rejected": -0.14363731443881989,
      "sft_loss": 1.3259879350662231,
      "step": 1210
    },
    {
      "epoch": 1.9721155789048292,
      "grad_norm": 2.2601141929626465,
      "learning_rate": 1.309278581280791e-06,
      "logits/chosen": -20.75266456604004,
      "logits/rejected": -20.531200408935547,
      "logps/chosen": -1.2139203548431396,
      "logps/rejected": -1.3907318115234375,
      "loss": 1.2807,
      "odds_ratio_loss": 0.6679332852363586,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12139203399419785,
      "rewards/margins": 0.017681140452623367,
      "rewards/rejected": -0.1390731781721115,
      "sft_loss": 1.2139203548431396,
      "step": 1220
    },
    {
      "epoch": 1.9882804606991311,
      "grad_norm": 1.351995825767517,
      "learning_rate": 1.272200556913199e-06,
      "logits/chosen": -20.665508270263672,
      "logits/rejected": -20.715679168701172,
      "logps/chosen": -1.2477220296859741,
      "logps/rejected": -1.4040950536727905,
      "loss": 1.3204,
      "odds_ratio_loss": 0.7266369462013245,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12477221339941025,
      "rewards/margins": 0.015637310221791267,
      "rewards/rejected": -0.14040951430797577,
      "sft_loss": 1.2477220296859741,
      "step": 1230
    },
    {
      "epoch": 2.004445342493433,
      "grad_norm": 3.1234052181243896,
      "learning_rate": 1.2354751015877698e-06,
      "logits/chosen": -20.652090072631836,
      "logits/rejected": -20.866174697875977,
      "logps/chosen": -1.1843925714492798,
      "logps/rejected": -1.4374680519104004,
      "loss": 1.2509,
      "odds_ratio_loss": 0.6650271415710449,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11843927204608917,
      "rewards/margins": 0.02530754543840885,
      "rewards/rejected": -0.14374680817127228,
      "sft_loss": 1.1843925714492798,
      "step": 1240
    },
    {
      "epoch": 2.020610224287735,
      "grad_norm": 2.8965847492218018,
      "learning_rate": 1.1991127612113945e-06,
      "logits/chosen": -20.925878524780273,
      "logits/rejected": -21.110370635986328,
      "logps/chosen": -1.2498103380203247,
      "logps/rejected": -1.4370229244232178,
      "loss": 1.3151,
      "odds_ratio_loss": 0.6532084345817566,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12498103082180023,
      "rewards/margins": 0.018721258267760277,
      "rewards/rejected": -0.14370229840278625,
      "sft_loss": 1.2498103380203247,
      "step": 1250
    },
    {
      "epoch": 2.036775106082037,
      "grad_norm": 2.9547386169433594,
      "learning_rate": 1.1631239774206035e-06,
      "logits/chosen": -20.5147705078125,
      "logits/rejected": -20.376428604125977,
      "logps/chosen": -1.236487627029419,
      "logps/rejected": -1.4057966470718384,
      "loss": 1.3074,
      "odds_ratio_loss": 0.7092265486717224,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12364877760410309,
      "rewards/margins": 0.01693090796470642,
      "rewards/rejected": -0.14057967066764832,
      "sft_loss": 1.236487627029419,
      "step": 1260
    },
    {
      "epoch": 2.052939987876339,
      "grad_norm": 3.2073123455047607,
      "learning_rate": 1.1275190845831978e-06,
      "logits/chosen": -20.78199005126953,
      "logits/rejected": -20.76191520690918,
      "logps/chosen": -1.211749792098999,
      "logps/rejected": -1.4196996688842773,
      "loss": 1.2743,
      "odds_ratio_loss": 0.6256077289581299,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12117497622966766,
      "rewards/margins": 0.020794976502656937,
      "rewards/rejected": -0.1419699490070343,
      "sft_loss": 1.211749792098999,
      "step": 1270
    },
    {
      "epoch": 2.0691048696706407,
      "grad_norm": 1.841145634651184,
      "learning_rate": 1.0923083068306778e-06,
      "logits/chosen": -20.74366569519043,
      "logits/rejected": -20.469417572021484,
      "logps/chosen": -1.1939375400543213,
      "logps/rejected": -1.4483907222747803,
      "loss": 1.2578,
      "odds_ratio_loss": 0.6386052966117859,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11939376592636108,
      "rewards/margins": 0.025445321574807167,
      "rewards/rejected": -0.1448390930891037,
      "sft_loss": 1.1939375400543213,
      "step": 1280
    },
    {
      "epoch": 2.0852697514649425,
      "grad_norm": 1.7583280801773071,
      "learning_rate": 1.0575017551223348e-06,
      "logits/chosen": -20.574234008789062,
      "logits/rejected": -20.410676956176758,
      "logps/chosen": -1.1411545276641846,
      "logps/rejected": -1.3073662519454956,
      "loss": 1.2096,
      "odds_ratio_loss": 0.6839541792869568,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11411545425653458,
      "rewards/margins": 0.016621166840195656,
      "rewards/rejected": -0.13073661923408508,
      "sft_loss": 1.1411545276641846,
      "step": 1290
    },
    {
      "epoch": 2.1014346332592444,
      "grad_norm": 1.8893145322799683,
      "learning_rate": 1.023109424341833e-06,
      "logits/chosen": -20.674358367919922,
      "logits/rejected": -20.841875076293945,
      "logps/chosen": -1.318074107170105,
      "logps/rejected": -1.4704669713974,
      "loss": 1.3891,
      "odds_ratio_loss": 0.7104289531707764,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13180741667747498,
      "rewards/margins": 0.01523929089307785,
      "rewards/rejected": -0.14704671502113342,
      "sft_loss": 1.318074107170105,
      "step": 1300
    },
    {
      "epoch": 2.1175995150535463,
      "grad_norm": 1.844677209854126,
      "learning_rate": 9.891411904271273e-07,
      "logits/chosen": -20.51424217224121,
      "logits/rejected": -20.641857147216797,
      "logps/chosen": -1.168316125869751,
      "logps/rejected": -1.3018604516983032,
      "loss": 1.2364,
      "odds_ratio_loss": 0.6810965538024902,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11683161556720734,
      "rewards/margins": 0.013354445807635784,
      "rewards/rejected": -0.1301860511302948,
      "sft_loss": 1.168316125869751,
      "step": 1310
    },
    {
      "epoch": 2.133764396847848,
      "grad_norm": 1.6055060625076294,
      "learning_rate": 9.556068075345363e-07,
      "logits/chosen": -20.898950576782227,
      "logits/rejected": -20.616153717041016,
      "logps/chosen": -1.2051537036895752,
      "logps/rejected": -1.2989599704742432,
      "loss": 1.2733,
      "odds_ratio_loss": 0.6817353367805481,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1205153614282608,
      "rewards/margins": 0.009380649775266647,
      "rewards/rejected": -0.12989601492881775,
      "sft_loss": 1.2051537036895752,
      "step": 1320
    },
    {
      "epoch": 2.14992927864215,
      "grad_norm": 1.451566219329834,
      "learning_rate": 9.225159052377838e-07,
      "logits/chosen": -20.857812881469727,
      "logits/rejected": -20.955089569091797,
      "logps/chosen": -1.242044448852539,
      "logps/rejected": -1.4105781316757202,
      "loss": 1.311,
      "odds_ratio_loss": 0.6894499063491821,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12420444190502167,
      "rewards/margins": 0.016853367909789085,
      "rewards/rejected": -0.1410578191280365,
      "sft_loss": 1.242044448852539,
      "step": 1330
    },
    {
      "epoch": 2.166094160436452,
      "grad_norm": 3.742938995361328,
      "learning_rate": 8.898779857628184e-07,
      "logits/chosen": -20.516098022460938,
      "logits/rejected": -20.78036117553711,
      "logps/chosen": -1.1591819524765015,
      "logps/rejected": -1.2938919067382812,
      "loss": 1.2271,
      "odds_ratio_loss": 0.6796503663063049,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11591820418834686,
      "rewards/margins": 0.013470989651978016,
      "rewards/rejected": -0.1293891966342926,
      "sft_loss": 1.1591819524765015,
      "step": 1340
    },
    {
      "epoch": 2.1822590422307537,
      "grad_norm": 1.6822633743286133,
      "learning_rate": 8.577024212591975e-07,
      "logits/chosen": -21.10406494140625,
      "logits/rejected": -20.88094711303711,
      "logps/chosen": -1.2675734758377075,
      "logps/rejected": -1.3737549781799316,
      "loss": 1.3373,
      "odds_ratio_loss": 0.6970787048339844,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12675733864307404,
      "rewards/margins": 0.010618140920996666,
      "rewards/rejected": -0.13737548887729645,
      "sft_loss": 1.2675734758377075,
      "step": 1350
    },
    {
      "epoch": 2.1984239240250556,
      "grad_norm": 2.0817370414733887,
      "learning_rate": 8.259984511088276e-07,
      "logits/chosen": -20.762319564819336,
      "logits/rejected": -20.748157501220703,
      "logps/chosen": -1.2708746194839478,
      "logps/rejected": -1.3757803440093994,
      "loss": 1.3444,
      "odds_ratio_loss": 0.7355143427848816,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12708747386932373,
      "rewards/margins": 0.010490546002984047,
      "rewards/rejected": -0.13757804036140442,
      "sft_loss": 1.2708746194839478,
      "step": 1360
    },
    {
      "epoch": 2.2145888058193575,
      "grad_norm": 1.3843287229537964,
      "learning_rate": 7.947751792728237e-07,
      "logits/chosen": -20.849414825439453,
      "logits/rejected": -20.718679428100586,
      "logps/chosen": -1.1914303302764893,
      "logps/rejected": -1.3572627305984497,
      "loss": 1.257,
      "odds_ratio_loss": 0.6554330587387085,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.119143046438694,
      "rewards/margins": 0.016583234071731567,
      "rewards/rejected": -0.13572627305984497,
      "sft_loss": 1.1914303302764893,
      "step": 1370
    },
    {
      "epoch": 2.2307536876136593,
      "grad_norm": 10.062128067016602,
      "learning_rate": 7.640415716772626e-07,
      "logits/chosen": -20.725990295410156,
      "logits/rejected": -20.761856079101562,
      "logps/chosen": -1.3113725185394287,
      "logps/rejected": -1.4960176944732666,
      "loss": 1.381,
      "odds_ratio_loss": 0.6963750123977661,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13113726675510406,
      "rewards/margins": 0.01846454106271267,
      "rewards/rejected": -0.14960181713104248,
      "sft_loss": 1.3113725185394287,
      "step": 1380
    },
    {
      "epoch": 2.246918569407961,
      "grad_norm": 2.1556107997894287,
      "learning_rate": 7.338064536385722e-07,
      "logits/chosen": -20.7343692779541,
      "logits/rejected": -20.6729793548584,
      "logps/chosen": -1.2534866333007812,
      "logps/rejected": -1.4300428628921509,
      "loss": 1.3205,
      "odds_ratio_loss": 0.6699201464653015,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12534865736961365,
      "rewards/margins": 0.017655618488788605,
      "rewards/rejected": -0.14300426840782166,
      "sft_loss": 1.2534866333007812,
      "step": 1390
    },
    {
      "epoch": 2.263083451202263,
      "grad_norm": 2.0938661098480225,
      "learning_rate": 7.040785073292883e-07,
      "logits/chosen": -20.498939514160156,
      "logits/rejected": -20.509479522705078,
      "logps/chosen": -1.3050354719161987,
      "logps/rejected": -1.4407126903533936,
      "loss": 1.3786,
      "odds_ratio_loss": 0.7356770038604736,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1305035650730133,
      "rewards/margins": 0.013567718677222729,
      "rewards/rejected": -0.14407126605510712,
      "sft_loss": 1.3050354719161987,
      "step": 1400
    },
    {
      "epoch": 2.279248332996565,
      "grad_norm": 2.8200390338897705,
      "learning_rate": 6.748662692849297e-07,
      "logits/chosen": -20.943716049194336,
      "logits/rejected": -20.97843360900879,
      "logps/chosen": -1.2242540121078491,
      "logps/rejected": -1.4535945653915405,
      "loss": 1.2894,
      "odds_ratio_loss": 0.6512330770492554,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12242539972066879,
      "rewards/margins": 0.02293405309319496,
      "rewards/rejected": -0.14535947144031525,
      "sft_loss": 1.2242540121078491,
      "step": 1410
    },
    {
      "epoch": 2.295413214790867,
      "grad_norm": 3.1267454624176025,
      "learning_rate": 6.46178127952686e-07,
      "logits/chosen": -20.665185928344727,
      "logits/rejected": -20.615243911743164,
      "logps/chosen": -1.2219477891921997,
      "logps/rejected": -1.3721575736999512,
      "loss": 1.2881,
      "odds_ratio_loss": 0.6616476774215698,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12219478189945221,
      "rewards/margins": 0.015020984224975109,
      "rewards/rejected": -0.1372157633304596,
      "sft_loss": 1.2219477891921997,
      "step": 1420
    },
    {
      "epoch": 2.3115780965851687,
      "grad_norm": 2.4645252227783203,
      "learning_rate": 6.180223212826289e-07,
      "logits/chosen": -20.52808380126953,
      "logits/rejected": -20.359468460083008,
      "logps/chosen": -1.2114827632904053,
      "logps/rejected": -1.3234493732452393,
      "loss": 1.2822,
      "odds_ratio_loss": 0.7067694664001465,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12114828824996948,
      "rewards/margins": 0.011196643114089966,
      "rewards/rejected": -0.13234491646289825,
      "sft_loss": 1.2114827632904053,
      "step": 1430
    },
    {
      "epoch": 2.3277429783794705,
      "grad_norm": 1.1290780305862427,
      "learning_rate": 5.904069343621443e-07,
      "logits/chosen": -20.84099006652832,
      "logits/rejected": -20.68367576599121,
      "logps/chosen": -1.1913143396377563,
      "logps/rejected": -1.3442729711532593,
      "loss": 1.2568,
      "odds_ratio_loss": 0.6552363634109497,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1191314235329628,
      "rewards/margins": 0.015295890159904957,
      "rewards/rejected": -0.13442730903625488,
      "sft_loss": 1.1913143396377563,
      "step": 1440
    },
    {
      "epoch": 2.3439078601737724,
      "grad_norm": 4.884983539581299,
      "learning_rate": 5.633398970942544e-07,
      "logits/chosen": -20.582796096801758,
      "logits/rejected": -20.532047271728516,
      "logps/chosen": -1.2106900215148926,
      "logps/rejected": -1.3333368301391602,
      "loss": 1.2794,
      "odds_ratio_loss": 0.6871799826622009,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12106901407241821,
      "rewards/margins": 0.012264668941497803,
      "rewards/rejected": -0.13333368301391602,
      "sft_loss": 1.2106900215148926,
      "step": 1450
    },
    {
      "epoch": 2.3600727419680743,
      "grad_norm": 2.4621684551239014,
      "learning_rate": 5.368289819205069e-07,
      "logits/chosen": -20.69870376586914,
      "logits/rejected": -20.650863647460938,
      "logps/chosen": -1.1822137832641602,
      "logps/rejected": -1.2959617376327515,
      "loss": 1.2522,
      "odds_ratio_loss": 0.6997024416923523,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.11822137981653214,
      "rewards/margins": 0.011374806985259056,
      "rewards/rejected": -0.12959617376327515,
      "sft_loss": 1.1822137832641602,
      "step": 1460
    },
    {
      "epoch": 2.376237623762376,
      "grad_norm": 2.461702585220337,
      "learning_rate": 5.108818015890785e-07,
      "logits/chosen": -20.82188606262207,
      "logits/rejected": -20.84579086303711,
      "logps/chosen": -1.279842734336853,
      "logps/rejected": -1.449312448501587,
      "loss": 1.348,
      "odds_ratio_loss": 0.6815730333328247,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12798430025577545,
      "rewards/margins": 0.016946956515312195,
      "rewards/rejected": -0.14493125677108765,
      "sft_loss": 1.279842734336853,
      "step": 1470
    },
    {
      "epoch": 2.392402505556678,
      "grad_norm": 2.5109152793884277,
      "learning_rate": 4.855058069687291e-07,
      "logits/chosen": -20.60469627380371,
      "logits/rejected": -20.50313949584961,
      "logps/chosen": -1.1870944499969482,
      "logps/rejected": -1.382088541984558,
      "loss": 1.254,
      "odds_ratio_loss": 0.6693797707557678,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.11870943009853363,
      "rewards/margins": 0.01949942298233509,
      "rewards/rejected": -0.13820883631706238,
      "sft_loss": 1.1870944499969482,
      "step": 1480
    },
    {
      "epoch": 2.40856738735098,
      "grad_norm": 2.1160638332366943,
      "learning_rate": 4.607082849092523e-07,
      "logits/chosen": -20.56966781616211,
      "logits/rejected": -20.43010902404785,
      "logps/chosen": -1.324727177619934,
      "logps/rejected": -1.4259936809539795,
      "loss": 1.3968,
      "odds_ratio_loss": 0.7211604118347168,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1324726939201355,
      "rewards/margins": 0.010126668028533459,
      "rewards/rejected": -0.14259937405586243,
      "sft_loss": 1.324727177619934,
      "step": 1490
    },
    {
      "epoch": 2.4247322691452817,
      "grad_norm": 4.8063836097717285,
      "learning_rate": 4.3649635614901405e-07,
      "logits/chosen": -20.565994262695312,
      "logits/rejected": -20.88812255859375,
      "logps/chosen": -1.2172143459320068,
      "logps/rejected": -1.277565836906433,
      "loss": 1.2883,
      "odds_ratio_loss": 0.711229145526886,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12172143161296844,
      "rewards/margins": 0.006035161670297384,
      "rewards/rejected": -0.12775658071041107,
      "sft_loss": 1.2172143459320068,
      "step": 1500
    },
    {
      "epoch": 2.4247322691452817,
      "eval_logits/chosen": -20.66438865661621,
      "eval_logits/rejected": -20.695892333984375,
      "eval_logps/chosen": -1.2573115825653076,
      "eval_logps/rejected": -1.3996058702468872,
      "eval_loss": 1.3293771743774414,
      "eval_odds_ratio_loss": 0.7206552624702454,
      "eval_rewards/accuracies": 0.5345454812049866,
      "eval_rewards/chosen": -0.12573117017745972,
      "eval_rewards/margins": 0.014229409396648407,
      "eval_rewards/rejected": -0.13996057212352753,
      "eval_runtime": 82.3734,
      "eval_samples_per_second": 13.354,
      "eval_sft_loss": 1.2573115825653076,
      "eval_steps_per_second": 6.677,
      "step": 1500
    },
    {
      "epoch": 2.4408971509395836,
      "grad_norm": 2.092519998550415,
      "learning_rate": 4.128769732701973e-07,
      "logits/chosen": -20.61777114868164,
      "logits/rejected": -20.48794174194336,
      "logps/chosen": -1.2248725891113281,
      "logps/rejected": -1.3724851608276367,
      "loss": 1.2928,
      "odds_ratio_loss": 0.6797080039978027,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12248726189136505,
      "rewards/margins": 0.014761263504624367,
      "rewards/rejected": -0.13724853098392487,
      "sft_loss": 1.2248725891113281,
      "step": 1510
    },
    {
      "epoch": 2.4570620327338855,
      "grad_norm": 2.445197582244873,
      "learning_rate": 3.8985691870233046e-07,
      "logits/chosen": -20.743959426879883,
      "logits/rejected": -20.582164764404297,
      "logps/chosen": -1.2659637928009033,
      "logps/rejected": -1.4508241415023804,
      "loss": 1.3339,
      "odds_ratio_loss": 0.678980827331543,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1265963912010193,
      "rewards/margins": 0.01848602667450905,
      "rewards/rejected": -0.14508241415023804,
      "sft_loss": 1.2659637928009033,
      "step": 1520
    },
    {
      "epoch": 2.4732269145281873,
      "grad_norm": 6.312464714050293,
      "learning_rate": 3.6744280277467904e-07,
      "logits/chosen": -20.964641571044922,
      "logits/rejected": -21.01685333251953,
      "logps/chosen": -1.2571816444396973,
      "logps/rejected": -1.413922905921936,
      "loss": 1.3289,
      "odds_ratio_loss": 0.7169492244720459,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12571817636489868,
      "rewards/margins": 0.015674125403165817,
      "rewards/rejected": -0.1413923054933548,
      "sft_loss": 1.2571816444396973,
      "step": 1530
    },
    {
      "epoch": 2.489391796322489,
      "grad_norm": 2.195627450942993,
      "learning_rate": 3.456410618180503e-07,
      "logits/chosen": -19.93832778930664,
      "logits/rejected": -20.330801010131836,
      "logps/chosen": -1.1545674800872803,
      "logps/rejected": -1.4308401346206665,
      "loss": 1.2195,
      "odds_ratio_loss": 0.649765133857727,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.11545674502849579,
      "rewards/margins": 0.027627255767583847,
      "rewards/rejected": -0.14308401942253113,
      "sft_loss": 1.1545674800872803,
      "step": 1540
    },
    {
      "epoch": 2.5055566781167915,
      "grad_norm": 1.8385835886001587,
      "learning_rate": 3.244579563165753e-07,
      "logits/chosen": -20.591102600097656,
      "logits/rejected": -20.71021270751953,
      "logps/chosen": -1.2204065322875977,
      "logps/rejected": -1.452178955078125,
      "loss": 1.2861,
      "odds_ratio_loss": 0.65655916929245,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12204064428806305,
      "rewards/margins": 0.02317725121974945,
      "rewards/rejected": -0.1452178955078125,
      "sft_loss": 1.2204065322875977,
      "step": 1550
    },
    {
      "epoch": 2.521721559911093,
      "grad_norm": 1.5258744955062866,
      "learning_rate": 3.038995691099697e-07,
      "logits/chosen": -20.90871238708496,
      "logits/rejected": -20.73806381225586,
      "logps/chosen": -1.288413166999817,
      "logps/rejected": -1.4578988552093506,
      "loss": 1.36,
      "odds_ratio_loss": 0.7158418297767639,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1288413256406784,
      "rewards/margins": 0.016948552802205086,
      "rewards/rejected": -0.14578989148139954,
      "sft_loss": 1.288413166999817,
      "step": 1560
    },
    {
      "epoch": 2.5378864417053952,
      "grad_norm": 2.4840152263641357,
      "learning_rate": 2.839718036468192e-07,
      "logits/chosen": -20.923898696899414,
      "logits/rejected": -20.989917755126953,
      "logps/chosen": -1.3150464296340942,
      "logps/rejected": -1.4090582132339478,
      "loss": 1.3879,
      "odds_ratio_loss": 0.7283130288124084,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13150465488433838,
      "rewards/margins": 0.009401168674230576,
      "rewards/rejected": -0.14090582728385925,
      "sft_loss": 1.3150464296340942,
      "step": 1570
    },
    {
      "epoch": 2.5540513234996967,
      "grad_norm": 4.045677185058594,
      "learning_rate": 2.646803822893723e-07,
      "logits/chosen": -20.986186981201172,
      "logits/rejected": -20.914697647094727,
      "logps/chosen": -1.2622355222702026,
      "logps/rejected": -1.387225866317749,
      "loss": 1.3346,
      "odds_ratio_loss": 0.7240074872970581,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12622354924678802,
      "rewards/margins": 0.012499034404754639,
      "rewards/rejected": -0.13872259855270386,
      "sft_loss": 1.2622355222702026,
      "step": 1580
    },
    {
      "epoch": 2.570216205293999,
      "grad_norm": 1.6737409830093384,
      "learning_rate": 2.460308446703341e-07,
      "logits/chosen": -20.74363136291504,
      "logits/rejected": -20.45053482055664,
      "logps/chosen": -1.2218042612075806,
      "logps/rejected": -1.310227394104004,
      "loss": 1.293,
      "odds_ratio_loss": 0.7117012739181519,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12218043953180313,
      "rewards/margins": 0.008842318318784237,
      "rewards/rejected": -0.13102273643016815,
      "sft_loss": 1.2218042612075806,
      "step": 1590
    },
    {
      "epoch": 2.5863810870883004,
      "grad_norm": 2.487335443496704,
      "learning_rate": 2.2802854610213143e-07,
      "logits/chosen": -20.622318267822266,
      "logits/rejected": -20.454730987548828,
      "logps/chosen": -1.184971570968628,
      "logps/rejected": -1.4719953536987305,
      "loss": 1.2481,
      "odds_ratio_loss": 0.6310954689979553,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.11849715560674667,
      "rewards/margins": 0.02870238944888115,
      "rewards/rejected": -0.14719954133033752,
      "sft_loss": 1.184971570968628,
      "step": 1600
    },
    {
      "epoch": 2.6025459688826027,
      "grad_norm": 7.610108375549316,
      "learning_rate": 2.106786560391072e-07,
      "logits/chosen": -20.3998966217041,
      "logits/rejected": -20.51192855834961,
      "logps/chosen": -1.2781306505203247,
      "logps/rejected": -1.3927931785583496,
      "loss": 1.3503,
      "odds_ratio_loss": 0.7214857339859009,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12781307101249695,
      "rewards/margins": 0.011466264724731445,
      "rewards/rejected": -0.139279305934906,
      "sft_loss": 1.2781306505203247,
      "step": 1610
    },
    {
      "epoch": 2.6187108506769046,
      "grad_norm": 2.2551307678222656,
      "learning_rate": 1.9398615659308255e-07,
      "logits/chosen": -20.576255798339844,
      "logits/rejected": -20.769161224365234,
      "logps/chosen": -1.2140244245529175,
      "logps/rejected": -1.2931668758392334,
      "loss": 1.2827,
      "odds_ratio_loss": 0.6866000294685364,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12140244245529175,
      "rewards/margins": 0.007914242334663868,
      "rewards/rejected": -0.12931668758392334,
      "sft_loss": 1.2140244245529175,
      "step": 1620
    },
    {
      "epoch": 2.6348757324712064,
      "grad_norm": 2.147873640060425,
      "learning_rate": 1.7795584110272184e-07,
      "logits/chosen": -20.9218807220459,
      "logits/rejected": -20.841930389404297,
      "logps/chosen": -1.261558175086975,
      "logps/rejected": -1.3861316442489624,
      "loss": 1.3318,
      "odds_ratio_loss": 0.7022627592086792,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1261558085680008,
      "rewards/margins": 0.01245735865086317,
      "rewards/rejected": -0.13861316442489624,
      "sft_loss": 1.261558175086975,
      "step": 1630
    },
    {
      "epoch": 2.6510406142655083,
      "grad_norm": 6.040190696716309,
      "learning_rate": 1.6259231275709636e-07,
      "logits/chosen": -20.989246368408203,
      "logits/rejected": -20.884021759033203,
      "logps/chosen": -1.2266334295272827,
      "logps/rejected": -1.2976648807525635,
      "loss": 1.3008,
      "odds_ratio_loss": 0.7414904832839966,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12266335636377335,
      "rewards/margins": 0.007103143725544214,
      "rewards/rejected": -0.12976649403572083,
      "sft_loss": 1.2266334295272827,
      "step": 1640
    },
    {
      "epoch": 2.66720549605981,
      "grad_norm": 1.8157051801681519,
      "learning_rate": 1.478999832738548e-07,
      "logits/chosen": -20.997966766357422,
      "logits/rejected": -20.761661529541016,
      "logps/chosen": -1.2066887617111206,
      "logps/rejected": -1.385434865951538,
      "loss": 1.276,
      "odds_ratio_loss": 0.6930679082870483,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12066888809204102,
      "rewards/margins": 0.017874618992209435,
      "rewards/rejected": -0.138543501496315,
      "sft_loss": 1.2066887617111206,
      "step": 1650
    },
    {
      "epoch": 2.683370377854112,
      "grad_norm": 2.9221081733703613,
      "learning_rate": 1.338830716323769e-07,
      "logits/chosen": -20.6229248046875,
      "logits/rejected": -20.6382999420166,
      "logps/chosen": -1.2255455255508423,
      "logps/rejected": -1.3001086711883545,
      "loss": 1.2979,
      "odds_ratio_loss": 0.7236055135726929,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12255455553531647,
      "rewards/margins": 0.007456316612660885,
      "rewards/rejected": -0.13001087307929993,
      "sft_loss": 1.2255455255508423,
      "step": 1660
    },
    {
      "epoch": 2.699535259648414,
      "grad_norm": 3.0684990882873535,
      "learning_rate": 1.205456028622723e-07,
      "logits/chosen": -20.87864112854004,
      "logits/rejected": -20.77214241027832,
      "logps/chosen": -1.1658565998077393,
      "logps/rejected": -1.406781554222107,
      "loss": 1.2305,
      "odds_ratio_loss": 0.6469210386276245,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.11658565700054169,
      "rewards/margins": 0.024092501029372215,
      "rewards/rejected": -0.14067815244197845,
      "sft_loss": 1.1658565998077393,
      "step": 1670
    },
    {
      "epoch": 2.7157001414427158,
      "grad_norm": 1.2988094091415405,
      "learning_rate": 1.0789140688756805e-07,
      "logits/chosen": -21.001827239990234,
      "logits/rejected": -20.79971694946289,
      "logps/chosen": -1.2181146144866943,
      "logps/rejected": -1.434340238571167,
      "loss": 1.2805,
      "odds_ratio_loss": 0.6233752369880676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12181148678064346,
      "rewards/margins": 0.0216225478798151,
      "rewards/rejected": -0.14343401789665222,
      "sft_loss": 1.2181146144866943,
      "step": 1680
    },
    {
      "epoch": 2.7318650232370176,
      "grad_norm": 4.973554611206055,
      "learning_rate": 9.592411742693098e-08,
      "logits/chosen": -20.57479476928711,
      "logits/rejected": -20.399307250976562,
      "logps/chosen": -1.2814916372299194,
      "logps/rejected": -1.3428285121917725,
      "loss": 1.3583,
      "odds_ratio_loss": 0.7676435112953186,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12814916670322418,
      "rewards/margins": 0.006133680697530508,
      "rewards/rejected": -0.13428285717964172,
      "sft_loss": 1.2814916372299194,
      "step": 1690
    },
    {
      "epoch": 2.7480299050313195,
      "grad_norm": 1.3830405473709106,
      "learning_rate": 8.464717095022168e-08,
      "logits/chosen": -20.695613861083984,
      "logits/rejected": -20.391788482666016,
      "logps/chosen": -1.1915602684020996,
      "logps/rejected": -1.3796567916870117,
      "loss": 1.2587,
      "odds_ratio_loss": 0.671554684638977,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.11915602535009384,
      "rewards/margins": 0.01880965754389763,
      "rewards/rejected": -0.13796567916870117,
      "sft_loss": 1.1915602684020996,
      "step": 1700
    },
    {
      "epoch": 2.7641947868256214,
      "grad_norm": 2.5777227878570557,
      "learning_rate": 7.406380569169841e-08,
      "logits/chosen": -20.552873611450195,
      "logits/rejected": -20.59903335571289,
      "logps/chosen": -1.2640081644058228,
      "logps/rejected": -1.334334373474121,
      "loss": 1.3366,
      "odds_ratio_loss": 0.7264189720153809,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12640079855918884,
      "rewards/margins": 0.0070326318964362144,
      "rewards/rejected": -0.13343344628810883,
      "sft_loss": 1.2640081644058228,
      "step": 1710
    },
    {
      "epoch": 2.7803596686199232,
      "grad_norm": 10.019956588745117,
      "learning_rate": 6.417706072013808e-08,
      "logits/chosen": -20.83199119567871,
      "logits/rejected": -21.08270263671875,
      "logps/chosen": -1.1969940662384033,
      "logps/rejected": -1.326841115951538,
      "loss": 1.2649,
      "odds_ratio_loss": 0.6789032816886902,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11969941854476929,
      "rewards/margins": 0.012984703294932842,
      "rewards/rejected": -0.1326841115951538,
      "sft_loss": 1.1969940662384033,
      "step": 1720
    },
    {
      "epoch": 2.796524550414225,
      "grad_norm": 6.8160624504089355,
      "learning_rate": 5.498977506615294e-08,
      "logits/chosen": -20.777956008911133,
      "logits/rejected": -20.86117172241211,
      "logps/chosen": -1.2800042629241943,
      "logps/rejected": -1.3593509197235107,
      "loss": 1.3517,
      "odds_ratio_loss": 0.7168198227882385,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1280004382133484,
      "rewards/margins": 0.007934670895338058,
      "rewards/rejected": -0.13593511283397675,
      "sft_loss": 1.2800042629241943,
      "step": 1730
    },
    {
      "epoch": 2.812689432208527,
      "grad_norm": 1.6938509941101074,
      "learning_rate": 4.6504586906947756e-08,
      "logits/chosen": -21.01276206970215,
      "logits/rejected": -21.02105140686035,
      "logps/chosen": -1.2667747735977173,
      "logps/rejected": -1.3524465560913086,
      "loss": 1.3367,
      "odds_ratio_loss": 0.6990799903869629,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.126677468419075,
      "rewards/margins": 0.008567174896597862,
      "rewards/rejected": -0.13524465262889862,
      "sft_loss": 1.2667747735977173,
      "step": 1740
    },
    {
      "epoch": 2.828854314002829,
      "grad_norm": 6.788274765014648,
      "learning_rate": 3.8723932808754914e-08,
      "logits/chosen": -21.133237838745117,
      "logits/rejected": -21.116573333740234,
      "logps/chosen": -1.3402550220489502,
      "logps/rejected": -1.3987007141113281,
      "loss": 1.4126,
      "odds_ratio_loss": 0.7239315509796143,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13402549922466278,
      "rewards/margins": 0.005844591651111841,
      "rewards/rejected": -0.1398700773715973,
      "sft_loss": 1.3402550220489502,
      "step": 1750
    },
    {
      "epoch": 2.8450191957971307,
      "grad_norm": 4.773237228393555,
      "learning_rate": 3.1650047027158014e-08,
      "logits/chosen": -20.84109115600586,
      "logits/rejected": -21.033550262451172,
      "logps/chosen": -1.2243554592132568,
      "logps/rejected": -1.3495168685913086,
      "loss": 1.2905,
      "odds_ratio_loss": 0.6615921258926392,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12243553251028061,
      "rewards/margins": 0.012516150251030922,
      "rewards/rejected": -0.13495168089866638,
      "sft_loss": 1.2243554592132568,
      "step": 1760
    },
    {
      "epoch": 2.8611840775914326,
      "grad_norm": 1.8300281763076782,
      "learning_rate": 2.5284960865517848e-08,
      "logits/chosen": -20.63343048095703,
      "logits/rejected": -20.709888458251953,
      "logps/chosen": -1.1276973485946655,
      "logps/rejected": -1.3769396543502808,
      "loss": 1.1906,
      "odds_ratio_loss": 0.6287329196929932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11276974529027939,
      "rewards/margins": 0.024924224242568016,
      "rewards/rejected": -0.13769397139549255,
      "sft_loss": 1.1276973485946655,
      "step": 1770
    },
    {
      "epoch": 2.8773489593857344,
      "grad_norm": 1.8408336639404297,
      "learning_rate": 1.9630502091670388e-08,
      "logits/chosen": -20.521255493164062,
      "logits/rejected": -20.47439956665039,
      "logps/chosen": -1.2427160739898682,
      "logps/rejected": -1.4044514894485474,
      "loss": 1.3062,
      "odds_ratio_loss": 0.6351091265678406,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12427163124084473,
      "rewards/margins": 0.016173530369997025,
      "rewards/rejected": -0.14044514298439026,
      "sft_loss": 1.2427160739898682,
      "step": 1780
    },
    {
      "epoch": 2.8935138411800363,
      "grad_norm": 5.715972423553467,
      "learning_rate": 1.4688294413074677e-08,
      "logits/chosen": -20.392711639404297,
      "logits/rejected": -20.409814834594727,
      "logps/chosen": -1.12639582157135,
      "logps/rejected": -1.3582656383514404,
      "loss": 1.193,
      "odds_ratio_loss": 0.6661497354507446,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11263958364725113,
      "rewards/margins": 0.023186981678009033,
      "rewards/rejected": -0.13582655787467957,
      "sft_loss": 1.12639582157135,
      "step": 1790
    },
    {
      "epoch": 2.909678722974338,
      "grad_norm": 1.500075340270996,
      "learning_rate": 1.0459757010556626e-08,
      "logits/chosen": -20.747310638427734,
      "logits/rejected": -20.764245986938477,
      "logps/chosen": -1.218435287475586,
      "logps/rejected": -1.2865406274795532,
      "loss": 1.2909,
      "odds_ratio_loss": 0.724606990814209,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.12184351682662964,
      "rewards/margins": 0.0068105412647128105,
      "rewards/rejected": -0.12865407764911652,
      "sft_loss": 1.218435287475586,
      "step": 1800
    },
    {
      "epoch": 2.92584360476864,
      "grad_norm": 4.831416606903076,
      "learning_rate": 6.94610413078306e-09,
      "logits/chosen": -20.301036834716797,
      "logits/rejected": -20.699962615966797,
      "logps/chosen": -1.2750685214996338,
      "logps/rejected": -1.4494614601135254,
      "loss": 1.3451,
      "odds_ratio_loss": 0.7007848024368286,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12750686705112457,
      "rewards/margins": 0.017439283430576324,
      "rewards/rejected": -0.1449461281299591,
      "sft_loss": 1.2750685214996338,
      "step": 1810
    },
    {
      "epoch": 2.942008486562942,
      "grad_norm": 3.175490617752075,
      "learning_rate": 4.14834473758563e-09,
      "logits/chosen": -20.20795249938965,
      "logits/rejected": -20.349489212036133,
      "logps/chosen": -1.168878197669983,
      "logps/rejected": -1.3752342462539673,
      "loss": 1.2352,
      "odds_ratio_loss": 0.6636675596237183,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11688783019781113,
      "rewards/margins": 0.020635612308979034,
      "rewards/rejected": -0.13752342760562897,
      "sft_loss": 1.168878197669983,
      "step": 1820
    },
    {
      "epoch": 2.9581733683572438,
      "grad_norm": 1.267177700996399,
      "learning_rate": 2.067282222230349e-09,
      "logits/chosen": -20.883831024169922,
      "logits/rejected": -21.049150466918945,
      "logps/chosen": -1.19767427444458,
      "logps/rejected": -1.4529333114624023,
      "loss": 1.2618,
      "odds_ratio_loss": 0.6415713429450989,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1197674423456192,
      "rewards/margins": 0.02552589774131775,
      "rewards/rejected": -0.14529332518577576,
      "sft_loss": 1.19767427444458,
      "step": 1830
    },
    {
      "epoch": 2.9743382501515456,
      "grad_norm": 6.299037456512451,
      "learning_rate": 7.035141727212979e-10,
      "logits/chosen": -20.69939422607422,
      "logits/rejected": -20.93973159790039,
      "logps/chosen": -1.1146414279937744,
      "logps/rejected": -1.2947094440460205,
      "loss": 1.1819,
      "odds_ratio_loss": 0.6721066236495972,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11146414279937744,
      "rewards/margins": 0.018006790429353714,
      "rewards/rejected": -0.12947094440460205,
      "sft_loss": 1.1146414279937744,
      "step": 1840
    },
    {
      "epoch": 2.9905031319458475,
      "grad_norm": 7.900111675262451,
      "learning_rate": 5.743220219761592e-11,
      "logits/chosen": -20.688940048217773,
      "logits/rejected": -20.652080535888672,
      "logps/chosen": -1.2903720140457153,
      "logps/rejected": -1.3897130489349365,
      "loss": 1.3642,
      "odds_ratio_loss": 0.7377926111221313,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12903721630573273,
      "rewards/margins": 0.00993410311639309,
      "rewards/rejected": -0.13897129893302917,
      "sft_loss": 1.2903720140457153,
      "step": 1850
    },
    {
      "epoch": 2.9969690846635686,
      "step": 1854,
      "total_flos": 5.618252880760013e+17,
      "train_loss": 1.3892074464594277,
      "train_runtime": 8722.5011,
      "train_samples_per_second": 3.404,
      "train_steps_per_second": 0.213
    }
  ],
  "logging_steps": 10,
  "max_steps": 1854,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 5.618252880760013e+17,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}