{
  "best_metric": 0.8505691885948181,
  "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.3/lora/orpo-salt-half/checkpoint-1500",
  "epoch": 2.9974597798475866,
  "eval_steps": 500,
  "global_step": 1770,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01693480101608806,
      "grad_norm": 8.384780883789062,
      "learning_rate": 4.999614014035063e-06,
      "logits/chosen": -2.9335973262786865,
      "logits/rejected": -2.9718575477600098,
      "logps/chosen": -1.0935328006744385,
      "logps/rejected": -1.5320154428482056,
      "loss": 1.1602,
      "odds_ratio_loss": 0.6671324968338013,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.10935328900814056,
      "rewards/margins": 0.04384826496243477,
      "rewards/rejected": -0.15320155024528503,
      "sft_loss": 1.0935328006744385,
      "step": 10
    },
    {
      "epoch": 0.03386960203217612,
      "grad_norm": 5.07673454284668,
      "learning_rate": 4.998440543386042e-06,
      "logits/chosen": -2.899705648422241,
      "logits/rejected": -2.8991312980651855,
      "logps/chosen": -1.0815098285675049,
      "logps/rejected": -1.5599451065063477,
      "loss": 1.1396,
      "odds_ratio_loss": 0.5810434818267822,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10815098136663437,
      "rewards/margins": 0.04784352704882622,
      "rewards/rejected": -0.1559945046901703,
      "sft_loss": 1.0815098285675049,
      "step": 20
    },
    {
      "epoch": 0.05080440304826418,
      "grad_norm": 4.302060127258301,
      "learning_rate": 4.996479918381253e-06,
      "logits/chosen": -2.944632053375244,
      "logits/rejected": -2.975834846496582,
      "logps/chosen": -1.056970238685608,
      "logps/rejected": -1.281690001487732,
      "loss": 1.1267,
      "odds_ratio_loss": 0.6977876424789429,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.10569703578948975,
      "rewards/margins": 0.022471977397799492,
      "rewards/rejected": -0.1281690150499344,
      "sft_loss": 1.056970238685608,
      "step": 30
    },
    {
      "epoch": 0.06773920406435224,
      "grad_norm": 3.221508026123047,
      "learning_rate": 4.993732756731818e-06,
      "logits/chosen": -2.925711154937744,
      "logits/rejected": -2.972975730895996,
      "logps/chosen": -0.8646748661994934,
      "logps/rejected": -1.3530806303024292,
      "loss": 0.9216,
      "odds_ratio_loss": 0.5692964792251587,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08646748960018158,
      "rewards/margins": 0.04884058237075806,
      "rewards/rejected": -0.13530807197093964,
      "sft_loss": 0.8646748661994934,
      "step": 40
    },
    {
      "epoch": 0.0846740050804403,
      "grad_norm": 11.413677215576172,
      "learning_rate": 4.9901999239537345e-06,
      "logits/chosen": -2.9809317588806152,
      "logits/rejected": -2.983664035797119,
      "logps/chosen": -0.9910508990287781,
      "logps/rejected": -1.344020962715149,
      "loss": 1.0558,
      "odds_ratio_loss": 0.6479853987693787,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0991050973534584,
      "rewards/margins": 0.035297006368637085,
      "rewards/rejected": -0.1344021111726761,
      "sft_loss": 0.9910508990287781,
      "step": 50
    },
    {
      "epoch": 0.10160880609652836,
      "grad_norm": 1.7445276975631714,
      "learning_rate": 4.985882533095186e-06,
      "logits/chosen": -2.9525933265686035,
      "logits/rejected": -2.955658435821533,
      "logps/chosen": -0.9359370470046997,
      "logps/rejected": -1.2753798961639404,
      "loss": 1.0023,
      "odds_ratio_loss": 0.6634014248847961,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09359370172023773,
      "rewards/margins": 0.03394431248307228,
      "rewards/rejected": -0.12753799557685852,
      "sft_loss": 0.9359370470046997,
      "step": 60
    },
    {
      "epoch": 0.11854360711261643,
      "grad_norm": 3.4384100437164307,
      "learning_rate": 4.9807819443858705e-06,
      "logits/chosen": -2.956145763397217,
      "logits/rejected": -2.9750401973724365,
      "logps/chosen": -0.8652639389038086,
      "logps/rejected": -1.2253761291503906,
      "loss": 0.9227,
      "odds_ratio_loss": 0.5748167634010315,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08652639389038086,
      "rewards/margins": 0.0360112190246582,
      "rewards/rejected": -0.12253761291503906,
      "sft_loss": 0.8652639389038086,
      "step": 70
    },
    {
      "epoch": 0.1354784081287045,
      "grad_norm": 2.6011013984680176,
      "learning_rate": 4.9748997648084404e-06,
      "logits/chosen": -2.927016496658325,
      "logits/rejected": -2.961350917816162,
      "logps/chosen": -0.9362251162528992,
      "logps/rejected": -1.1120071411132812,
      "loss": 1.0068,
      "odds_ratio_loss": 0.7062079906463623,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09362251311540604,
      "rewards/margins": 0.017578203231096268,
      "rewards/rejected": -0.111200712621212,
      "sft_loss": 0.9362251162528992,
      "step": 80
    },
    {
      "epoch": 0.15241320914479256,
      "grad_norm": 2.413475751876831,
      "learning_rate": 4.96823784759222e-06,
      "logits/chosen": -2.869145154953003,
      "logits/rejected": -2.916862964630127,
      "logps/chosen": -0.8781692385673523,
      "logps/rejected": -1.2194924354553223,
      "loss": 0.9334,
      "odds_ratio_loss": 0.552145779132843,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08781692385673523,
      "rewards/margins": 0.034132327884435654,
      "rewards/rejected": -0.12194924056529999,
      "sft_loss": 0.8781692385673523,
      "step": 90
    },
    {
      "epoch": 0.1693480101608806,
      "grad_norm": 1.9281786680221558,
      "learning_rate": 4.960798291629323e-06,
      "logits/chosen": -2.9603519439697266,
      "logits/rejected": -2.9720394611358643,
      "logps/chosen": -0.8728249669075012,
      "logps/rejected": -1.0510112047195435,
      "loss": 0.9387,
      "odds_ratio_loss": 0.658331036567688,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08728249371051788,
      "rewards/margins": 0.017818626016378403,
      "rewards/rejected": -0.10510112345218658,
      "sft_loss": 0.8728249669075012,
      "step": 100
    },
    {
      "epoch": 0.18628281117696868,
      "grad_norm": 2.8832967281341553,
      "learning_rate": 4.952583440813383e-06,
      "logits/chosen": -2.9769511222839355,
      "logits/rejected": -2.991858720779419,
      "logps/chosen": -0.9942096471786499,
      "logps/rejected": -1.25453519821167,
      "loss": 1.0667,
      "odds_ratio_loss": 0.7251978516578674,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09942097961902618,
      "rewards/margins": 0.026032552123069763,
      "rewards/rejected": -0.12545353174209595,
      "sft_loss": 0.9942096471786499,
      "step": 110
    },
    {
      "epoch": 0.20321761219305673,
      "grad_norm": 1.2281321287155151,
      "learning_rate": 4.943595883301086e-06,
      "logits/chosen": -2.9461495876312256,
      "logits/rejected": -2.988556385040283,
      "logps/chosen": -0.8882713317871094,
      "logps/rejected": -1.1232919692993164,
      "loss": 0.9522,
      "odds_ratio_loss": 0.6390038728713989,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08882713317871094,
      "rewards/margins": 0.02350207045674324,
      "rewards/rejected": -0.11232920736074448,
      "sft_loss": 0.8882713317871094,
      "step": 120
    },
    {
      "epoch": 0.2201524132091448,
      "grad_norm": 2.2648568153381348,
      "learning_rate": 4.933838450696757e-06,
      "logits/chosen": -2.9817254543304443,
      "logits/rejected": -3.0018677711486816,
      "logps/chosen": -0.8987566828727722,
      "logps/rejected": -1.0799884796142578,
      "loss": 0.9707,
      "odds_ratio_loss": 0.7198494672775269,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08987566828727722,
      "rewards/margins": 0.018123187124729156,
      "rewards/rejected": -0.10799884796142578,
      "sft_loss": 0.8987566828727722,
      "step": 130
    },
    {
      "epoch": 0.23708721422523285,
      "grad_norm": 3.667508840560913,
      "learning_rate": 4.923314217160234e-06,
      "logits/chosen": -2.988133192062378,
      "logits/rejected": -3.033163070678711,
      "logps/chosen": -0.9288301467895508,
      "logps/rejected": -1.1756031513214111,
      "loss": 0.9973,
      "odds_ratio_loss": 0.6846222877502441,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.09288302063941956,
      "rewards/margins": 0.024677302688360214,
      "rewards/rejected": -0.11756031215190887,
      "sft_loss": 0.9288301467895508,
      "step": 140
    },
    {
      "epoch": 0.2540220152413209,
      "grad_norm": 1.4715845584869385,
      "learning_rate": 4.9120264984383285e-06,
      "logits/chosen": -2.997676134109497,
      "logits/rejected": -2.9935667514801025,
      "logps/chosen": -0.8239797353744507,
      "logps/rejected": -0.9983611106872559,
      "loss": 0.8869,
      "odds_ratio_loss": 0.6288361549377441,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08239796757698059,
      "rewards/margins": 0.017438137903809547,
      "rewards/rejected": -0.09983611106872559,
      "sft_loss": 0.8239797353744507,
      "step": 150
    },
    {
      "epoch": 0.270956816257409,
      "grad_norm": 1.3789633512496948,
      "learning_rate": 4.899978850820176e-06,
      "logits/chosen": -3.004080295562744,
      "logits/rejected": -3.0049595832824707,
      "logps/chosen": -0.9372480511665344,
      "logps/rejected": -1.147570252418518,
      "loss": 1.0008,
      "odds_ratio_loss": 0.635485053062439,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0937248021364212,
      "rewards/margins": 0.021032210439443588,
      "rewards/rejected": -0.11475701630115509,
      "sft_loss": 0.9372480511665344,
      "step": 160
    },
    {
      "epoch": 0.28789161727349705,
      "grad_norm": 2.1128056049346924,
      "learning_rate": 4.887175070016795e-06,
      "logits/chosen": -3.027259349822998,
      "logits/rejected": -3.0453531742095947,
      "logps/chosen": -0.7962668538093567,
      "logps/rejected": -1.0727083683013916,
      "loss": 0.8585,
      "odds_ratio_loss": 0.6222477555274963,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07962668687105179,
      "rewards/margins": 0.027644142508506775,
      "rewards/rejected": -0.10727082192897797,
      "sft_loss": 0.7962668538093567,
      "step": 170
    },
    {
      "epoch": 0.3048264182895851,
      "grad_norm": 4.011683464050293,
      "learning_rate": 4.873619189965217e-06,
      "logits/chosen": -2.973634719848633,
      "logits/rejected": -2.989631414413452,
      "logps/chosen": -0.8350585103034973,
      "logps/rejected": -1.051026463508606,
      "loss": 0.9,
      "odds_ratio_loss": 0.6493778824806213,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08350586146116257,
      "rewards/margins": 0.021596785634756088,
      "rewards/rejected": -0.10510264337062836,
      "sft_loss": 0.8350585103034973,
      "step": 180
    },
    {
      "epoch": 0.32176121930567314,
      "grad_norm": 1.2796882390975952,
      "learning_rate": 4.859315481557563e-06,
      "logits/chosen": -2.9721431732177734,
      "logits/rejected": -2.9849448204040527,
      "logps/chosen": -0.821880042552948,
      "logps/rejected": -1.0275905132293701,
      "loss": 0.8867,
      "odds_ratio_loss": 0.648421585559845,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08218801021575928,
      "rewards/margins": 0.0205710306763649,
      "rewards/rejected": -0.10275904089212418,
      "sft_loss": 0.821880042552948,
      "step": 190
    },
    {
      "epoch": 0.3386960203217612,
      "grad_norm": 1.8488657474517822,
      "learning_rate": 4.84426845129546e-06,
      "logits/chosen": -3.0222136974334717,
      "logits/rejected": -3.031564235687256,
      "logps/chosen": -0.8574220538139343,
      "logps/rejected": -0.9888635873794556,
      "loss": 0.9247,
      "odds_ratio_loss": 0.6725481748580933,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08574221283197403,
      "rewards/margins": 0.013144141063094139,
      "rewards/rejected": -0.09888636320829391,
      "sft_loss": 0.8574220538139343,
      "step": 200
    },
    {
      "epoch": 0.3556308213378493,
      "grad_norm": 1.5179060697555542,
      "learning_rate": 4.828482839870233e-06,
      "logits/chosen": -3.010849714279175,
      "logits/rejected": -3.0288453102111816,
      "logps/chosen": -0.8511277437210083,
      "logps/rejected": -0.9780422449111938,
      "loss": 0.9215,
      "odds_ratio_loss": 0.7037326693534851,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08511276543140411,
      "rewards/margins": 0.012691453099250793,
      "rewards/rejected": -0.0978042259812355,
      "sft_loss": 0.8511277437210083,
      "step": 210
    },
    {
      "epoch": 0.37256562235393736,
      "grad_norm": 1.2089393138885498,
      "learning_rate": 4.811963620669314e-06,
      "logits/chosen": -3.0581696033477783,
      "logits/rejected": -3.099684476852417,
      "logps/chosen": -0.8706866502761841,
      "logps/rejected": -1.0248219966888428,
      "loss": 0.9377,
      "odds_ratio_loss": 0.6702378988265991,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08706866204738617,
      "rewards/margins": 0.015413539484143257,
      "rewards/rejected": -0.10248219966888428,
      "sft_loss": 0.8706866502761841,
      "step": 220
    },
    {
      "epoch": 0.3895004233700254,
      "grad_norm": 2.3151497840881348,
      "learning_rate": 4.794715998209328e-06,
      "logits/chosen": -2.9359335899353027,
      "logits/rejected": -2.9507675170898438,
      "logps/chosen": -0.8457564115524292,
      "logps/rejected": -1.0168864727020264,
      "loss": 0.9116,
      "odds_ratio_loss": 0.6584563255310059,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08457564562559128,
      "rewards/margins": 0.017113011330366135,
      "rewards/rejected": -0.10168864578008652,
      "sft_loss": 0.8457564115524292,
      "step": 230
    },
    {
      "epoch": 0.40643522438611346,
      "grad_norm": 2.3710029125213623,
      "learning_rate": 4.7767454064963724e-06,
      "logits/chosen": -2.9894230365753174,
      "logits/rejected": -3.022892475128174,
      "logps/chosen": -0.8459660410881042,
      "logps/rejected": -1.015751838684082,
      "loss": 0.9112,
      "odds_ratio_loss": 0.6523627042770386,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08459659665822983,
      "rewards/margins": 0.016978587955236435,
      "rewards/rejected": -0.10157518088817596,
      "sft_loss": 0.8459660410881042,
      "step": 240
    },
    {
      "epoch": 0.42337002540220153,
      "grad_norm": 2.1464998722076416,
      "learning_rate": 4.758057507313987e-06,
      "logits/chosen": -3.024229049682617,
      "logits/rejected": -3.034865617752075,
      "logps/chosen": -0.8055688142776489,
      "logps/rejected": -1.001123070716858,
      "loss": 0.869,
      "odds_ratio_loss": 0.6340595483779907,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08055686950683594,
      "rewards/margins": 0.019555436447262764,
      "rewards/rejected": -0.10011231899261475,
      "sft_loss": 0.8055688142776489,
      "step": 250
    },
    {
      "epoch": 0.4403048264182896,
      "grad_norm": 3.3724582195281982,
      "learning_rate": 4.73865818843936e-06,
      "logits/chosen": -3.042086124420166,
      "logits/rejected": -3.0638880729675293,
      "logps/chosen": -0.912920355796814,
      "logps/rejected": -1.1694436073303223,
      "loss": 0.9795,
      "odds_ratio_loss": 0.6657846570014954,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09129203855991364,
      "rewards/margins": 0.025652330368757248,
      "rewards/rejected": -0.11694437265396118,
      "sft_loss": 0.912920355796814,
      "step": 260
    },
    {
      "epoch": 0.4572396274343776,
      "grad_norm": 1.6793408393859863,
      "learning_rate": 4.718553561788339e-06,
      "logits/chosen": -3.0312306880950928,
      "logits/rejected": -3.060295820236206,
      "logps/chosen": -0.836656928062439,
      "logps/rejected": -0.986899733543396,
      "loss": 0.9016,
      "odds_ratio_loss": 0.6491778492927551,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08366570621728897,
      "rewards/margins": 0.015024276450276375,
      "rewards/rejected": -0.0986899808049202,
      "sft_loss": 0.836656928062439,
      "step": 270
    },
    {
      "epoch": 0.4741744284504657,
      "grad_norm": 2.6274185180664062,
      "learning_rate": 4.697749961489822e-06,
      "logits/chosen": -3.061577558517456,
      "logits/rejected": -3.0702619552612305,
      "logps/chosen": -0.8848905563354492,
      "logps/rejected": -1.0645692348480225,
      "loss": 0.9531,
      "odds_ratio_loss": 0.6824666857719421,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08848904818296432,
      "rewards/margins": 0.017967868596315384,
      "rewards/rejected": -0.10645692050457001,
      "sft_loss": 0.8848905563354492,
      "step": 280
    },
    {
      "epoch": 0.4911092294665538,
      "grad_norm": 2.089768886566162,
      "learning_rate": 4.67625394189013e-06,
      "logits/chosen": -3.0883841514587402,
      "logits/rejected": -3.090940475463867,
      "logps/chosen": -0.7822630405426025,
      "logps/rejected": -1.0061594247817993,
      "loss": 0.8425,
      "odds_ratio_loss": 0.6027771830558777,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07822629809379578,
      "rewards/margins": 0.022389648482203484,
      "rewards/rejected": -0.100615955889225,
      "sft_loss": 0.7822630405426025,
      "step": 290
    },
    {
      "epoch": 0.5080440304826418,
      "grad_norm": 2.2402989864349365,
      "learning_rate": 4.654072275488016e-06,
      "logits/chosen": -3.1045384407043457,
      "logits/rejected": -3.1042561531066895,
      "logps/chosen": -0.7530822157859802,
      "logps/rejected": -0.9303094744682312,
      "loss": 0.8152,
      "odds_ratio_loss": 0.621616542339325,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07530822604894638,
      "rewards/margins": 0.017722725868225098,
      "rewards/rejected": -0.09303095191717148,
      "sft_loss": 0.7530822157859802,
      "step": 300
    },
    {
      "epoch": 0.5249788314987299,
      "grad_norm": 7.859750747680664,
      "learning_rate": 4.631211950800925e-06,
      "logits/chosen": -3.0885701179504395,
      "logits/rejected": -3.1314892768859863,
      "logps/chosen": -0.8193107843399048,
      "logps/rejected": -1.0114855766296387,
      "loss": 0.8855,
      "odds_ratio_loss": 0.6622810959815979,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08193108439445496,
      "rewards/margins": 0.01921747997403145,
      "rewards/rejected": -0.1011485680937767,
      "sft_loss": 0.8193107843399048,
      "step": 310
    },
    {
      "epoch": 0.541913632514818,
      "grad_norm": 3.1328227519989014,
      "learning_rate": 4.6076801701632095e-06,
      "logits/chosen": -3.087184190750122,
      "logits/rejected": -3.1436970233917236,
      "logps/chosen": -0.801365852355957,
      "logps/rejected": -0.9046095609664917,
      "loss": 0.8686,
      "odds_ratio_loss": 0.6719350814819336,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08013658970594406,
      "rewards/margins": 0.010324367322027683,
      "rewards/rejected": -0.09046096354722977,
      "sft_loss": 0.801365852355957,
      "step": 320
    },
    {
      "epoch": 0.558848433530906,
      "grad_norm": 2.4766950607299805,
      "learning_rate": 4.583484347456972e-06,
      "logits/chosen": -3.0817341804504395,
      "logits/rejected": -3.114269256591797,
      "logps/chosen": -0.8467103242874146,
      "logps/rejected": -0.9794108271598816,
      "loss": 0.9136,
      "odds_ratio_loss": 0.6689561009407043,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0846710279583931,
      "rewards/margins": 0.013270048424601555,
      "rewards/rejected": -0.0979410782456398,
      "sft_loss": 0.8467103242874146,
      "step": 330
    },
    {
      "epoch": 0.5757832345469941,
      "grad_norm": 1.4625582695007324,
      "learning_rate": 4.55863210577626e-06,
      "logits/chosen": -3.123152256011963,
      "logits/rejected": -3.1361846923828125,
      "logps/chosen": -0.8682034611701965,
      "logps/rejected": -1.090419888496399,
      "loss": 0.9349,
      "odds_ratio_loss": 0.6673767566680908,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08682034909725189,
      "rewards/margins": 0.022221634164452553,
      "rewards/rejected": -0.10904198884963989,
      "sft_loss": 0.8682034611701965,
      "step": 340
    },
    {
      "epoch": 0.5927180355630821,
      "grad_norm": 1.6330158710479736,
      "learning_rate": 4.5331312750253465e-06,
      "logits/chosen": -3.079641103744507,
      "logits/rejected": -3.1221771240234375,
      "logps/chosen": -0.7776955962181091,
      "logps/rejected": -0.978277325630188,
      "loss": 0.8441,
      "odds_ratio_loss": 0.6638715267181396,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07776956260204315,
      "rewards/margins": 0.02005816623568535,
      "rewards/rejected": -0.0978277251124382,
      "sft_loss": 0.7776955962181091,
      "step": 350
    },
    {
      "epoch": 0.6096528365791702,
      "grad_norm": 1.9444369077682495,
      "learning_rate": 4.506989889451858e-06,
      "logits/chosen": -3.1103367805480957,
      "logits/rejected": -3.1434414386749268,
      "logps/chosen": -0.8445068597793579,
      "logps/rejected": -0.9869591593742371,
      "loss": 0.9108,
      "odds_ratio_loss": 0.662727952003479,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08445067703723907,
      "rewards/margins": 0.014245236292481422,
      "rewards/rejected": -0.09869591891765594,
      "sft_loss": 0.8445068597793579,
      "step": 360
    },
    {
      "epoch": 0.6265876375952583,
      "grad_norm": 2.251150369644165,
      "learning_rate": 4.480216185115512e-06,
      "logits/chosen": -3.0998404026031494,
      "logits/rejected": -3.087653398513794,
      "logps/chosen": -0.8066253662109375,
      "logps/rejected": -0.9696222543716431,
      "loss": 0.8704,
      "odds_ratio_loss": 0.6379208564758301,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08066253364086151,
      "rewards/margins": 0.016299689188599586,
      "rewards/rejected": -0.09696222841739655,
      "sft_loss": 0.8066253662109375,
      "step": 370
    },
    {
      "epoch": 0.6435224386113463,
      "grad_norm": 1.8241935968399048,
      "learning_rate": 4.4528185972932856e-06,
      "logits/chosen": -3.019221782684326,
      "logits/rejected": -3.083693265914917,
      "logps/chosen": -0.8054075241088867,
      "logps/rejected": -1.0394740104675293,
      "loss": 0.8728,
      "odds_ratio_loss": 0.6744040846824646,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08054076135158539,
      "rewards/margins": 0.023406637832522392,
      "rewards/rejected": -0.10394741594791412,
      "sft_loss": 0.8054075241088867,
      "step": 380
    },
    {
      "epoch": 0.6604572396274344,
      "grad_norm": 7.428930759429932,
      "learning_rate": 4.424805757821803e-06,
      "logits/chosen": -3.0259501934051514,
      "logits/rejected": -3.0692214965820312,
      "logps/chosen": -0.8603572845458984,
      "logps/rejected": -0.9688261151313782,
      "loss": 0.9297,
      "odds_ratio_loss": 0.6930958032608032,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08603573590517044,
      "rewards/margins": 0.010846875607967377,
      "rewards/rejected": -0.09688261151313782,
      "sft_loss": 0.8603572845458984,
      "step": 390
    },
    {
      "epoch": 0.6773920406435224,
      "grad_norm": 1.7724531888961792,
      "learning_rate": 4.396186492377812e-06,
      "logits/chosen": -3.049534797668457,
      "logits/rejected": -3.0946240425109863,
      "logps/chosen": -0.804740309715271,
      "logps/rejected": -1.0287699699401855,
      "loss": 0.8638,
      "odds_ratio_loss": 0.5903798341751099,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08047403395175934,
      "rewards/margins": 0.02240295708179474,
      "rewards/rejected": -0.10287699848413467,
      "sft_loss": 0.804740309715271,
      "step": 400
    },
    {
      "epoch": 0.6943268416596104,
      "grad_norm": 2.2618014812469482,
      "learning_rate": 4.366969817697578e-06,
      "logits/chosen": -3.0235114097595215,
      "logits/rejected": -3.083573341369629,
      "logps/chosen": -0.7958801984786987,
      "logps/rejected": -0.9750314950942993,
      "loss": 0.8637,
      "odds_ratio_loss": 0.6784581542015076,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07958801835775375,
      "rewards/margins": 0.01791512593626976,
      "rewards/rejected": -0.09750314056873322,
      "sft_loss": 0.7958801984786987,
      "step": 410
    },
    {
      "epoch": 0.7112616426756986,
      "grad_norm": 2.504657506942749,
      "learning_rate": 4.337164938736086e-06,
      "logits/chosen": -3.049118757247925,
      "logits/rejected": -3.05595326423645,
      "logps/chosen": -0.8237883448600769,
      "logps/rejected": -0.918091893196106,
      "loss": 0.8939,
      "odds_ratio_loss": 0.7009984254837036,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.08237884193658829,
      "rewards/margins": 0.00943034328520298,
      "rewards/rejected": -0.09180918335914612,
      "sft_loss": 0.8237883448600769,
      "step": 420
    },
    {
      "epoch": 0.7281964436917866,
      "grad_norm": 1.2601932287216187,
      "learning_rate": 4.306781245766945e-06,
      "logits/chosen": -3.0380876064300537,
      "logits/rejected": -3.0915472507476807,
      "logps/chosen": -0.7913134098052979,
      "logps/rejected": -1.0263410806655884,
      "loss": 0.8569,
      "odds_ratio_loss": 0.6553653478622437,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0791313424706459,
      "rewards/margins": 0.023502767086029053,
      "rewards/rejected": -0.10263410955667496,
      "sft_loss": 0.7913134098052979,
      "step": 430
    },
    {
      "epoch": 0.7451312447078747,
      "grad_norm": 4.184472560882568,
      "learning_rate": 4.275828311423903e-06,
      "logits/chosen": -3.093045473098755,
      "logits/rejected": -3.094749927520752,
      "logps/chosen": -0.8900951147079468,
      "logps/rejected": -0.9482099413871765,
      "loss": 0.9611,
      "odds_ratio_loss": 0.7105128765106201,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08900952339172363,
      "rewards/margins": 0.005811482667922974,
      "rewards/rejected": -0.09482099860906601,
      "sft_loss": 0.8900951147079468,
      "step": 440
    },
    {
      "epoch": 0.7620660457239627,
      "grad_norm": 5.296163082122803,
      "learning_rate": 4.244315887684912e-06,
      "logits/chosen": -3.063938617706299,
      "logits/rejected": -3.0689697265625,
      "logps/chosen": -0.7619583010673523,
      "logps/rejected": -0.9321343302726746,
      "loss": 0.8277,
      "odds_ratio_loss": 0.6576007008552551,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07619582116603851,
      "rewards/margins": 0.017017606645822525,
      "rewards/rejected": -0.09321344643831253,
      "sft_loss": 0.7619583010673523,
      "step": 450
    },
    {
      "epoch": 0.7790008467400508,
      "grad_norm": 1.0935202836990356,
      "learning_rate": 4.212253902799685e-06,
      "logits/chosen": -3.107138156890869,
      "logits/rejected": -3.1228718757629395,
      "logps/chosen": -0.828906238079071,
      "logps/rejected": -1.0854886770248413,
      "loss": 0.8936,
      "odds_ratio_loss": 0.6471681594848633,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08289062976837158,
      "rewards/margins": 0.02565823495388031,
      "rewards/rejected": -0.10854886472225189,
      "sft_loss": 0.828906238079071,
      "step": 460
    },
    {
      "epoch": 0.7959356477561389,
      "grad_norm": 2.112194538116455,
      "learning_rate": 4.179652458161718e-06,
      "logits/chosen": -3.0884511470794678,
      "logits/rejected": -3.086930751800537,
      "logps/chosen": -0.8770621418952942,
      "logps/rejected": -0.9670013189315796,
      "loss": 0.9476,
      "odds_ratio_loss": 0.7056951522827148,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08770622313022614,
      "rewards/margins": 0.008993919007480145,
      "rewards/rejected": -0.09670013934373856,
      "sft_loss": 0.8770621418952942,
      "step": 470
    },
    {
      "epoch": 0.8128704487722269,
      "grad_norm": 1.4803324937820435,
      "learning_rate": 4.146521825125765e-06,
      "logits/chosen": -3.1123909950256348,
      "logits/rejected": -3.127878189086914,
      "logps/chosen": -0.8552261590957642,
      "logps/rejected": -0.9841393232345581,
      "loss": 0.9226,
      "odds_ratio_loss": 0.6738197803497314,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0855226144194603,
      "rewards/margins": 0.012891319580376148,
      "rewards/rejected": -0.09841393679380417,
      "sft_loss": 0.8552261590957642,
      "step": 480
    },
    {
      "epoch": 0.8298052497883149,
      "grad_norm": 1.639560580253601,
      "learning_rate": 4.11287244177176e-06,
      "logits/chosen": -3.1404106616973877,
      "logits/rejected": -3.1268608570098877,
      "logps/chosen": -0.7970486283302307,
      "logps/rejected": -1.0359351634979248,
      "loss": 0.8576,
      "odds_ratio_loss": 0.6055651307106018,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07970486581325531,
      "rewards/margins": 0.02388865128159523,
      "rewards/rejected": -0.10359351336956024,
      "sft_loss": 0.7970486283302307,
      "step": 490
    },
    {
      "epoch": 0.8467400508044031,
      "grad_norm": 7.098996162414551,
      "learning_rate": 4.078714909616215e-06,
      "logits/chosen": -3.1334495544433594,
      "logits/rejected": -3.127417802810669,
      "logps/chosen": -0.8159440755844116,
      "logps/rejected": -1.052442193031311,
      "loss": 0.8758,
      "odds_ratio_loss": 0.5985492467880249,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08159441500902176,
      "rewards/margins": 0.02364981174468994,
      "rewards/rejected": -0.10524420440196991,
      "sft_loss": 0.8159440755844116,
      "step": 500
    },
    {
      "epoch": 0.8467400508044031,
      "eval_logits/chosen": -3.1088695526123047,
      "eval_logits/rejected": -3.1276028156280518,
      "eval_logps/chosen": -0.8053962588310242,
      "eval_logps/rejected": -1.0085769891738892,
      "eval_loss": 0.8691067099571228,
      "eval_odds_ratio_loss": 0.6371051073074341,
      "eval_rewards/accuracies": 0.5704761743545532,
      "eval_rewards/chosen": -0.08053962886333466,
      "eval_rewards/margins": 0.02031807415187359,
      "eval_rewards/rejected": -0.1008576974272728,
      "eval_runtime": 194.1916,
      "eval_samples_per_second": 5.407,
      "eval_sft_loss": 0.8053962588310242,
      "eval_steps_per_second": 2.704,
      "step": 500
    },
    {
      "epoch": 0.8636748518204911,
      "grad_norm": 5.1792216300964355,
      "learning_rate": 4.044059990272125e-06,
      "logits/chosen": -3.1320395469665527,
      "logits/rejected": -3.1584267616271973,
      "logps/chosen": -0.8421246409416199,
      "logps/rejected": -1.076226830482483,
      "loss": 0.9067,
      "odds_ratio_loss": 0.64554762840271,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08421246707439423,
      "rewards/margins": 0.02341020107269287,
      "rewards/rejected": -0.10762268304824829,
      "sft_loss": 0.8421246409416199,
      "step": 510
    },
    {
      "epoch": 0.8806096528365792,
      "grad_norm": 2.52073073387146,
      "learning_rate": 4.0089186020584345e-06,
      "logits/chosen": -3.1114721298217773,
      "logits/rejected": -3.1454081535339355,
      "logps/chosen": -0.8953489065170288,
      "logps/rejected": -1.0218431949615479,
      "loss": 0.9597,
      "odds_ratio_loss": 0.6433413028717041,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08953489363193512,
      "rewards/margins": 0.012649421580135822,
      "rewards/rejected": -0.10218431800603867,
      "sft_loss": 0.8953489065170288,
      "step": 520
    },
    {
      "epoch": 0.8975444538526672,
      "grad_norm": 3.2529776096343994,
      "learning_rate": 3.973301816560124e-06,
      "logits/chosen": -3.1164424419403076,
      "logits/rejected": -3.0959460735321045,
      "logps/chosen": -0.8098430633544922,
      "logps/rejected": -1.0107625722885132,
      "loss": 0.871,
      "odds_ratio_loss": 0.6116595268249512,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08098430931568146,
      "rewards/margins": 0.0200919471681118,
      "rewards/rejected": -0.10107626020908356,
      "sft_loss": 0.8098430633544922,
      "step": 530
    },
    {
      "epoch": 0.9144792548687553,
      "grad_norm": 2.4143612384796143,
      "learning_rate": 3.937220855140021e-06,
      "logits/chosen": -3.1310832500457764,
      "logits/rejected": -3.1453144550323486,
      "logps/chosen": -0.8188160061836243,
      "logps/rejected": -0.9051122665405273,
      "loss": 0.8891,
      "odds_ratio_loss": 0.7029509544372559,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08188159763813019,
      "rewards/margins": 0.008629636839032173,
      "rewards/rejected": -0.09051123261451721,
      "sft_loss": 0.8188160061836243,
      "step": 540
    },
    {
      "epoch": 0.9314140558848434,
      "grad_norm": 1.3889896869659424,
      "learning_rate": 3.900687085403418e-06,
      "logits/chosen": -3.1416361331939697,
      "logits/rejected": -3.169236421585083,
      "logps/chosen": -0.8118529319763184,
      "logps/rejected": -0.8874411582946777,
      "loss": 0.8817,
      "odds_ratio_loss": 0.6982238292694092,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08118529617786407,
      "rewards/margins": 0.007558824960142374,
      "rewards/rejected": -0.08874412626028061,
      "sft_loss": 0.8118529319763184,
      "step": 550
    },
    {
      "epoch": 0.9483488569009314,
      "grad_norm": 2.055143356323242,
      "learning_rate": 3.863712017616614e-06,
      "logits/chosen": -3.131472110748291,
      "logits/rejected": -3.160679340362549,
      "logps/chosen": -0.8101698160171509,
      "logps/rejected": -0.9761570692062378,
      "loss": 0.8764,
      "odds_ratio_loss": 0.6618725657463074,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08101697266101837,
      "rewards/margins": 0.016598742455244064,
      "rewards/rejected": -0.09761571884155273,
      "sft_loss": 0.8101698160171509,
      "step": 560
    },
    {
      "epoch": 0.9652836579170194,
      "grad_norm": 14.660100936889648,
      "learning_rate": 3.826307301080504e-06,
      "logits/chosen": -3.0291128158569336,
      "logits/rejected": -3.0481247901916504,
      "logps/chosen": -0.8100768327713013,
      "logps/rejected": -1.162418007850647,
      "loss": 0.8764,
      "odds_ratio_loss": 0.6627860069274902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08100768178701401,
      "rewards/margins": 0.03523411601781845,
      "rewards/rejected": -0.11624179035425186,
      "sft_loss": 0.8100768327713013,
      "step": 570
    },
    {
      "epoch": 0.9822184589331076,
      "grad_norm": 2.563514471054077,
      "learning_rate": 3.7884847204603775e-06,
      "logits/chosen": -3.1155200004577637,
      "logits/rejected": -3.15079665184021,
      "logps/chosen": -0.8116699457168579,
      "logps/rejected": -0.9339970350265503,
      "loss": 0.8812,
      "odds_ratio_loss": 0.6949950456619263,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08116699010133743,
      "rewards/margins": 0.012232715263962746,
      "rewards/rejected": -0.09339970350265503,
      "sft_loss": 0.8116699457168579,
      "step": 580
    },
    {
      "epoch": 0.9991532599491956,
      "grad_norm": 2.7657859325408936,
      "learning_rate": 3.750256192073058e-06,
      "logits/chosen": -3.172693967819214,
      "logits/rejected": -3.1799886226654053,
      "logps/chosen": -0.9269296526908875,
      "logps/rejected": -0.9760646820068359,
      "loss": 1.0009,
      "odds_ratio_loss": 0.7394477128982544,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.09269297122955322,
      "rewards/margins": 0.004913498647511005,
      "rewards/rejected": -0.09760646522045135,
      "sft_loss": 0.9269296526908875,
      "step": 590
    },
    {
      "epoch": 1.0160880609652836,
      "grad_norm": 7.485799789428711,
      "learning_rate": 3.7116337601325715e-06,
      "logits/chosen": -3.1055843830108643,
      "logits/rejected": -3.129669189453125,
      "logps/chosen": -0.7523837089538574,
      "logps/rejected": -0.8988542556762695,
      "loss": 0.8166,
      "odds_ratio_loss": 0.6426426768302917,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07523836940526962,
      "rewards/margins": 0.014647054485976696,
      "rewards/rejected": -0.08988542854785919,
      "sft_loss": 0.7523837089538574,
      "step": 600
    },
    {
      "epoch": 1.0330228619813717,
      "grad_norm": 2.2944157123565674,
      "learning_rate": 3.6726295929555154e-06,
      "logits/chosen": -3.077573776245117,
      "logits/rejected": -3.11928391456604,
      "logps/chosen": -0.7445582747459412,
      "logps/rejected": -0.9014002680778503,
      "loss": 0.8117,
      "odds_ratio_loss": 0.6712278723716736,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07445583492517471,
      "rewards/margins": 0.015684202313423157,
      "rewards/rejected": -0.09014002978801727,
      "sft_loss": 0.7445582747459412,
      "step": 610
    },
    {
      "epoch": 1.0499576629974599,
      "grad_norm": 2.4759469032287598,
      "learning_rate": 3.6332559791273307e-06,
      "logits/chosen": -3.070753812789917,
      "logits/rejected": -3.133881092071533,
      "logps/chosen": -0.7787492871284485,
      "logps/rejected": -0.9435569643974304,
      "loss": 0.843,
      "odds_ratio_loss": 0.6423131823539734,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07787492126226425,
      "rewards/margins": 0.016480756923556328,
      "rewards/rejected": -0.09435568749904633,
      "sft_loss": 0.7787492871284485,
      "step": 620
    },
    {
      "epoch": 1.0668924640135478,
      "grad_norm": 2.397857666015625,
      "learning_rate": 3.593525323630681e-06,
      "logits/chosen": -3.0695629119873047,
      "logits/rejected": -3.0961527824401855,
      "logps/chosen": -0.8072765469551086,
      "logps/rejected": -0.9367998838424683,
      "loss": 0.8715,
      "odds_ratio_loss": 0.6417396664619446,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08072765171527863,
      "rewards/margins": 0.012952342629432678,
      "rewards/rejected": -0.0936800017952919,
      "sft_loss": 0.8072765469551086,
      "step": 630
    },
    {
      "epoch": 1.083827265029636,
      "grad_norm": 2.2752127647399902,
      "learning_rate": 3.5534501439371615e-06,
      "logits/chosen": -3.1115057468414307,
      "logits/rejected": -3.1277289390563965,
      "logps/chosen": -0.7920553088188171,
      "logps/rejected": -0.9753093719482422,
      "loss": 0.8548,
      "odds_ratio_loss": 0.6271573901176453,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07920553535223007,
      "rewards/margins": 0.018325407058000565,
      "rewards/rejected": -0.09753094613552094,
      "sft_loss": 0.7920553088188171,
      "step": 640
    },
    {
      "epoch": 1.100762066045724,
      "grad_norm": 1.311848521232605,
      "learning_rate": 3.5130430660635633e-06,
      "logits/chosen": -3.1020348072052,
      "logits/rejected": -3.1430366039276123,
      "logps/chosen": -0.8032411336898804,
      "logps/rejected": -0.9853434562683105,
      "loss": 0.864,
      "odds_ratio_loss": 0.6071646809577942,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08032412827014923,
      "rewards/margins": 0.018210221081972122,
      "rewards/rejected": -0.09853433817625046,
      "sft_loss": 0.8032411336898804,
      "step": 650
    },
    {
      "epoch": 1.117696867061812,
      "grad_norm": 1.6412379741668701,
      "learning_rate": 3.4723168205939444e-06,
      "logits/chosen": -3.0930168628692627,
      "logits/rejected": -3.1322126388549805,
      "logps/chosen": -0.7815112471580505,
      "logps/rejected": -0.9042918086051941,
      "loss": 0.8483,
      "odds_ratio_loss": 0.6679799556732178,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07815112918615341,
      "rewards/margins": 0.012278061360120773,
      "rewards/rejected": -0.09042918682098389,
      "sft_loss": 0.7815112471580505,
      "step": 660
    },
    {
      "epoch": 1.1346316680779,
      "grad_norm": 2.1591804027557373,
      "learning_rate": 3.431284238668754e-06,
      "logits/chosen": -3.062398910522461,
      "logits/rejected": -3.1151247024536133,
      "logps/chosen": -0.8405235409736633,
      "logps/rejected": -1.0043154954910278,
      "loss": 0.9059,
      "odds_ratio_loss": 0.6538293957710266,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08405234664678574,
      "rewards/margins": 0.01637919992208481,
      "rewards/rejected": -0.10043156147003174,
      "sft_loss": 0.8405235409736633,
      "step": 670
    },
    {
      "epoch": 1.1515664690939882,
      "grad_norm": 4.811310291290283,
      "learning_rate": 3.389958247942274e-06,
      "logits/chosen": -3.057506561279297,
      "logits/rejected": -3.1269962787628174,
      "logps/chosen": -0.8411120176315308,
      "logps/rejected": -1.1102626323699951,
      "loss": 0.9091,
      "odds_ratio_loss": 0.6803519129753113,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08411119878292084,
      "rewards/margins": 0.026915064081549644,
      "rewards/rejected": -0.11102626472711563,
      "sft_loss": 0.8411120176315308,
      "step": 680
    },
    {
      "epoch": 1.168501270110076,
      "grad_norm": 2.754120111465454,
      "learning_rate": 3.3483518685096588e-06,
      "logits/chosen": -3.08880615234375,
      "logits/rejected": -3.109083890914917,
      "logps/chosen": -0.8396803140640259,
      "logps/rejected": -0.9775651693344116,
      "loss": 0.9066,
      "odds_ratio_loss": 0.6687448620796204,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08396803587675095,
      "rewards/margins": 0.013788496144115925,
      "rewards/rejected": -0.097756527364254,
      "sft_loss": 0.8396803140640259,
      "step": 690
    },
    {
      "epoch": 1.1854360711261642,
      "grad_norm": 1.9642765522003174,
      "learning_rate": 3.306478208804839e-06,
      "logits/chosen": -3.0659680366516113,
      "logits/rejected": -3.1026992797851562,
      "logps/chosen": -0.7790535092353821,
      "logps/rejected": -0.9482936859130859,
      "loss": 0.8451,
      "odds_ratio_loss": 0.659988284111023,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07790535688400269,
      "rewards/margins": 0.016924021765589714,
      "rewards/rejected": -0.09482936561107635,
      "sft_loss": 0.7790535092353821,
      "step": 700
    },
    {
      "epoch": 1.2023708721422524,
      "grad_norm": 2.6194725036621094,
      "learning_rate": 3.264350461470608e-06,
      "logits/chosen": -3.0373668670654297,
      "logits/rejected": -3.0688533782958984,
      "logps/chosen": -0.7216525077819824,
      "logps/rejected": -1.0178143978118896,
      "loss": 0.7814,
      "odds_ratio_loss": 0.597113311290741,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07216525077819824,
      "rewards/margins": 0.02961619198322296,
      "rewards/rejected": -0.1017814427614212,
      "sft_loss": 0.7216525077819824,
      "step": 710
    },
    {
      "epoch": 1.2193056731583405,
      "grad_norm": 2.312389373779297,
      "learning_rate": 3.2219818992021685e-06,
      "logits/chosen": -3.03488826751709,
      "logits/rejected": -3.087043285369873,
      "logps/chosen": -0.7246071696281433,
      "logps/rejected": -1.047911286354065,
      "loss": 0.7828,
      "odds_ratio_loss": 0.5823417901992798,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07246071845293045,
      "rewards/margins": 0.03233041241765022,
      "rewards/rejected": -0.10479112714529037,
      "sft_loss": 0.7246071696281433,
      "step": 720
    },
    {
      "epoch": 1.2362404741744284,
      "grad_norm": 7.234289169311523,
      "learning_rate": 3.1793858705654595e-06,
      "logits/chosen": -3.0948994159698486,
      "logits/rejected": -3.1080453395843506,
      "logps/chosen": -0.7130419611930847,
      "logps/rejected": -0.9181682467460632,
      "loss": 0.7733,
      "odds_ratio_loss": 0.6023129820823669,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07130420207977295,
      "rewards/margins": 0.020512625575065613,
      "rewards/rejected": -0.09181682765483856,
      "sft_loss": 0.7130419611930847,
      "step": 730
    },
    {
      "epoch": 1.2531752751905165,
      "grad_norm": 2.101661443710327,
      "learning_rate": 3.1365757957915787e-06,
      "logits/chosen": -3.0874876976013184,
      "logits/rejected": -3.123832941055298,
      "logps/chosen": -0.8178297877311707,
      "logps/rejected": -0.9593319892883301,
      "loss": 0.8832,
      "odds_ratio_loss": 0.6534532308578491,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.0817829817533493,
      "rewards/margins": 0.01415021438151598,
      "rewards/rejected": -0.09593319892883301,
      "sft_loss": 0.8178297877311707,
      "step": 740
    },
    {
      "epoch": 1.2701100762066047,
      "grad_norm": 3.2394182682037354,
      "learning_rate": 3.093565162548633e-06,
      "logits/chosen": -3.0456290245056152,
      "logits/rejected": -3.0783424377441406,
      "logps/chosen": -0.8610566854476929,
      "logps/rejected": -1.0578378438949585,
      "loss": 0.9282,
      "odds_ratio_loss": 0.6714938879013062,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08610567450523376,
      "rewards/margins": 0.019678115844726562,
      "rewards/rejected": -0.10578378289937973,
      "sft_loss": 0.8610566854476929,
      "step": 750
    },
    {
      "epoch": 1.2870448772226926,
      "grad_norm": 3.4462602138519287,
      "learning_rate": 3.0503675216923294e-06,
      "logits/chosen": -3.112204074859619,
      "logits/rejected": -3.1115825176239014,
      "logps/chosen": -0.7481369376182556,
      "logps/rejected": -0.9443623423576355,
      "loss": 0.8093,
      "odds_ratio_loss": 0.6117558479309082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07481369376182556,
      "rewards/margins": 0.019622545689344406,
      "rewards/rejected": -0.09443624317646027,
      "sft_loss": 0.7481369376182556,
      "step": 760
    },
    {
      "epoch": 1.3039796782387807,
      "grad_norm": 2.5497653484344482,
      "learning_rate": 3.0069964829966748e-06,
      "logits/chosen": -3.083761692047119,
      "logits/rejected": -3.1253674030303955,
      "logps/chosen": -0.7777606248855591,
      "logps/rejected": -0.9011049270629883,
      "loss": 0.844,
      "odds_ratio_loss": 0.6621277928352356,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07777605950832367,
      "rewards/margins": 0.01233443059027195,
      "rewards/rejected": -0.09011048078536987,
      "sft_loss": 0.7777606248855591,
      "step": 770
    },
    {
      "epoch": 1.3209144792548688,
      "grad_norm": 2.1115994453430176,
      "learning_rate": 2.963465710866094e-06,
      "logits/chosen": -3.098053455352783,
      "logits/rejected": -3.1173479557037354,
      "logps/chosen": -0.7589127421379089,
      "logps/rejected": -1.0739343166351318,
      "loss": 0.8183,
      "odds_ratio_loss": 0.5940018892288208,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07589127868413925,
      "rewards/margins": 0.03150214999914169,
      "rewards/rejected": -0.10739342123270035,
      "sft_loss": 0.7589127421379089,
      "step": 780
    },
    {
      "epoch": 1.337849280270957,
      "grad_norm": 7.320650577545166,
      "learning_rate": 2.919788920030357e-06,
      "logits/chosen": -3.143812894821167,
      "logits/rejected": -3.1576976776123047,
      "logps/chosen": -0.8158448934555054,
      "logps/rejected": -0.9654959440231323,
      "loss": 0.8826,
      "odds_ratio_loss": 0.6670835614204407,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08158449828624725,
      "rewards/margins": 0.014965096488595009,
      "rewards/rejected": -0.09654959291219711,
      "sft_loss": 0.8158448934555054,
      "step": 790
    },
    {
      "epoch": 1.3547840812870449,
      "grad_norm": 1.3826831579208374,
      "learning_rate": 2.8759798712236303e-06,
      "logits/chosen": -3.1377153396606445,
      "logits/rejected": -3.138549566268921,
      "logps/chosen": -0.7528073191642761,
      "logps/rejected": -1.0746774673461914,
      "loss": 0.8173,
      "odds_ratio_loss": 0.6448089480400085,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07528072595596313,
      "rewards/margins": 0.032187022268772125,
      "rewards/rejected": -0.10746775567531586,
      "sft_loss": 0.7528073191642761,
      "step": 800
    },
    {
      "epoch": 1.371718882303133,
      "grad_norm": 2.1327664852142334,
      "learning_rate": 2.8320523668490507e-06,
      "logits/chosen": -3.095376968383789,
      "logits/rejected": -3.129220485687256,
      "logps/chosen": -0.8166864514350891,
      "logps/rejected": -0.9957429766654968,
      "loss": 0.8841,
      "odds_ratio_loss": 0.6738199591636658,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08166865259408951,
      "rewards/margins": 0.01790565252304077,
      "rewards/rejected": -0.09957430511713028,
      "sft_loss": 0.8166864514350891,
      "step": 810
    },
    {
      "epoch": 1.388653683319221,
      "grad_norm": 2.5101046562194824,
      "learning_rate": 2.7880202466301597e-06,
      "logits/chosen": -3.075023651123047,
      "logits/rejected": -3.1098039150238037,
      "logps/chosen": -0.8058909177780151,
      "logps/rejected": -0.9401804208755493,
      "loss": 0.8735,
      "odds_ratio_loss": 0.6758350133895874,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08058909326791763,
      "rewards/margins": 0.013428950682282448,
      "rewards/rejected": -0.09401804953813553,
      "sft_loss": 0.8058909177780151,
      "step": 820
    },
    {
      "epoch": 1.405588484335309,
      "grad_norm": 2.202185869216919,
      "learning_rate": 2.7438973832505854e-06,
      "logits/chosen": -3.060824155807495,
      "logits/rejected": -3.0545971393585205,
      "logps/chosen": -0.7594717741012573,
      "logps/rejected": -0.9759091138839722,
      "loss": 0.8235,
      "odds_ratio_loss": 0.6401799321174622,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07594718039035797,
      "rewards/margins": 0.021643735468387604,
      "rewards/rejected": -0.09759090840816498,
      "sft_loss": 0.7594717741012573,
      "step": 830
    },
    {
      "epoch": 1.4225232853513972,
      "grad_norm": 5.100254058837891,
      "learning_rate": 2.699697677983341e-06,
      "logits/chosen": -3.1682240962982178,
      "logits/rejected": -3.182861804962158,
      "logps/chosen": -0.8106738924980164,
      "logps/rejected": -0.8703418970108032,
      "loss": 0.8831,
      "odds_ratio_loss": 0.7241007685661316,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08106739819049835,
      "rewards/margins": 0.0059667956084012985,
      "rewards/rejected": -0.0870341882109642,
      "sft_loss": 0.8106738924980164,
      "step": 840
    },
    {
      "epoch": 1.4394580863674853,
      "grad_norm": 7.667366981506348,
      "learning_rate": 2.6554350563111115e-06,
      "logits/chosen": -3.0851314067840576,
      "logits/rejected": -3.116276502609253,
      "logps/chosen": -0.8307350873947144,
      "logps/rejected": -0.9147516489028931,
      "loss": 0.9054,
      "odds_ratio_loss": 0.7461589574813843,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08307350426912308,
      "rewards/margins": 0.0084016602486372,
      "rewards/rejected": -0.09147517383098602,
      "sft_loss": 0.8307350873947144,
      "step": 850
    },
    {
      "epoch": 1.4563928873835732,
      "grad_norm": 6.640262126922607,
      "learning_rate": 2.611123463538913e-06,
      "logits/chosen": -3.0749902725219727,
      "logits/rejected": -3.0911591053009033,
      "logps/chosen": -0.7142345309257507,
      "logps/rejected": -0.9272225499153137,
      "loss": 0.7775,
      "odds_ratio_loss": 0.6324297189712524,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07142344862222672,
      "rewards/margins": 0.02129879966378212,
      "rewards/rejected": -0.09272225201129913,
      "sft_loss": 0.7142345309257507,
      "step": 860
    },
    {
      "epoch": 1.4733276883996613,
      "grad_norm": 8.2167329788208,
      "learning_rate": 2.566776860400514e-06,
      "logits/chosen": -3.1082987785339355,
      "logits/rejected": -3.1257712841033936,
      "logps/chosen": -0.8350755572319031,
      "logps/rejected": -1.056910753250122,
      "loss": 0.898,
      "odds_ratio_loss": 0.6290403604507446,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08350756019353867,
      "rewards/margins": 0.022183528169989586,
      "rewards/rejected": -0.10569107532501221,
      "sft_loss": 0.8350755572319031,
      "step": 870
    },
    {
      "epoch": 1.4902624894157492,
      "grad_norm": 1.4808557033538818,
      "learning_rate": 2.522409218659989e-06,
      "logits/chosen": -3.143266201019287,
      "logits/rejected": -3.170633316040039,
      "logps/chosen": -0.7854605913162231,
      "logps/rejected": -0.9068530797958374,
      "loss": 0.8512,
      "odds_ratio_loss": 0.6573347449302673,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07854606211185455,
      "rewards/margins": 0.012139257043600082,
      "rewards/rejected": -0.09068530797958374,
      "sft_loss": 0.7854605913162231,
      "step": 880
    },
    {
      "epoch": 1.5071972904318374,
      "grad_norm": 3.6185755729675293,
      "learning_rate": 2.4780345167097976e-06,
      "logits/chosen": -3.0972650051116943,
      "logits/rejected": -3.0835044384002686,
      "logps/chosen": -0.7852433919906616,
      "logps/rejected": -1.0855656862258911,
      "loss": 0.8459,
      "odds_ratio_loss": 0.6065645813941956,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07852433621883392,
      "rewards/margins": 0.030032237991690636,
      "rewards/rejected": -0.1085565835237503,
      "sft_loss": 0.7852433919906616,
      "step": 890
    },
    {
      "epoch": 1.5241320914479255,
      "grad_norm": 4.639472961425781,
      "learning_rate": 2.4336667351667747e-06,
      "logits/chosen": -3.114197015762329,
      "logits/rejected": -3.124145269393921,
      "logps/chosen": -0.8203206062316895,
      "logps/rejected": -1.0489810705184937,
      "loss": 0.8792,
      "odds_ratio_loss": 0.5887311697006226,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08203206956386566,
      "rewards/margins": 0.022866051644086838,
      "rewards/rejected": -0.1048981174826622,
      "sft_loss": 0.8203206062316895,
      "step": 900
    },
    {
      "epoch": 1.5410668924640136,
      "grad_norm": 3.153228282928467,
      "learning_rate": 2.3893198524674264e-06,
      "logits/chosen": -3.086516857147217,
      "logits/rejected": -3.104675769805908,
      "logps/chosen": -0.7846948504447937,
      "logps/rejected": -0.9825633764266968,
      "loss": 0.8478,
      "odds_ratio_loss": 0.6309666037559509,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07846949249505997,
      "rewards/margins": 0.019786860793828964,
      "rewards/rejected": -0.09825634956359863,
      "sft_loss": 0.7846948504447937,
      "step": 910
    },
    {
      "epoch": 1.5580016934801018,
      "grad_norm": 3.532456874847412,
      "learning_rate": 2.345007840463904e-06,
      "logits/chosen": -3.0608856678009033,
      "logits/rejected": -3.0965914726257324,
      "logps/chosen": -0.8089407682418823,
      "logps/rejected": -0.9399329423904419,
      "loss": 0.8759,
      "odds_ratio_loss": 0.6692665815353394,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08089407533407211,
      "rewards/margins": 0.013099217787384987,
      "rewards/rejected": -0.09399329125881195,
      "sft_loss": 0.8089407682418823,
      "step": 920
    },
    {
      "epoch": 1.5749364944961897,
      "grad_norm": 3.778723955154419,
      "learning_rate": 2.3007446600220572e-06,
      "logits/chosen": -3.122668504714966,
      "logits/rejected": -3.1052489280700684,
      "logps/chosen": -0.799084484577179,
      "logps/rejected": -0.9807415008544922,
      "loss": 0.8639,
      "odds_ratio_loss": 0.6483135223388672,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07990844547748566,
      "rewards/margins": 0.018165703862905502,
      "rewards/rejected": -0.09807415306568146,
      "sft_loss": 0.799084484577179,
      "step": 930
    },
    {
      "epoch": 1.5918712955122776,
      "grad_norm": 3.475341558456421,
      "learning_rate": 2.2565442566229507e-06,
      "logits/chosen": -3.0740559101104736,
      "logits/rejected": -3.106633424758911,
      "logps/chosen": -0.7921947240829468,
      "logps/rejected": -0.8962046504020691,
      "loss": 0.862,
      "odds_ratio_loss": 0.6985131502151489,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.07921947538852692,
      "rewards/margins": 0.01040099561214447,
      "rewards/rejected": -0.08962046355009079,
      "sft_loss": 0.7921947240829468,
      "step": 940
    },
    {
      "epoch": 1.6088060965283657,
      "grad_norm": 2.1668930053710938,
      "learning_rate": 2.2124205559692195e-06,
      "logits/chosen": -3.064317226409912,
      "logits/rejected": -3.1167845726013184,
      "logps/chosen": -0.7817971706390381,
      "logps/rejected": -1.0080362558364868,
      "loss": 0.8399,
      "odds_ratio_loss": 0.5807359218597412,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0781797245144844,
      "rewards/margins": 0.02262391336262226,
      "rewards/rejected": -0.10080362856388092,
      "sft_loss": 0.7817971706390381,
      "step": 950
    },
    {
      "epoch": 1.6257408975444538,
      "grad_norm": 1.8939071893692017,
      "learning_rate": 2.168387459597666e-06,
      "logits/chosen": -3.092590808868408,
      "logits/rejected": -3.138596773147583,
      "logps/chosen": -0.7956336140632629,
      "logps/rejected": -1.0347424745559692,
      "loss": 0.8565,
      "odds_ratio_loss": 0.6086810827255249,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07956336438655853,
      "rewards/margins": 0.02391088381409645,
      "rewards/rejected": -0.10347424447536469,
      "sft_loss": 0.7956336140632629,
      "step": 960
    },
    {
      "epoch": 1.642675698560542,
      "grad_norm": 2.1293139457702637,
      "learning_rate": 2.1244588404994648e-06,
      "logits/chosen": -3.0621867179870605,
      "logits/rejected": -3.0614800453186035,
      "logps/chosen": -0.7734932899475098,
      "logps/rejected": -0.9287079572677612,
      "loss": 0.8408,
      "odds_ratio_loss": 0.6727171540260315,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07734932750463486,
      "rewards/margins": 0.015521461144089699,
      "rewards/rejected": -0.0928707867860794,
      "sft_loss": 0.7734932899475098,
      "step": 970
    },
    {
      "epoch": 1.65961049957663,
      "grad_norm": 5.466368675231934,
      "learning_rate": 2.08064853874936e-06,
      "logits/chosen": -3.0892724990844727,
      "logits/rejected": -3.1403141021728516,
      "logps/chosen": -0.8142075538635254,
      "logps/rejected": -0.9884878396987915,
      "loss": 0.8786,
      "odds_ratio_loss": 0.6434410810470581,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08142076432704926,
      "rewards/margins": 0.017428018152713776,
      "rewards/rejected": -0.09884877502918243,
      "sft_loss": 0.8142075538635254,
      "step": 980
    },
    {
      "epoch": 1.676545300592718,
      "grad_norm": 12.251886367797852,
      "learning_rate": 2.0369703571452387e-06,
      "logits/chosen": -3.055989980697632,
      "logits/rejected": -3.0450901985168457,
      "logps/chosen": -0.716150164604187,
      "logps/rejected": -0.9780189394950867,
      "loss": 0.7743,
      "odds_ratio_loss": 0.5815138816833496,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07161502540111542,
      "rewards/margins": 0.026186879724264145,
      "rewards/rejected": -0.09780190140008926,
      "sft_loss": 0.716150164604187,
      "step": 990
    },
    {
      "epoch": 1.6934801016088061,
      "grad_norm": 3.6831836700439453,
      "learning_rate": 1.993438056859441e-06,
      "logits/chosen": -3.1155529022216797,
      "logits/rejected": -3.1060287952423096,
      "logps/chosen": -0.7484423518180847,
      "logps/rejected": -0.929480254650116,
      "loss": 0.8098,
      "odds_ratio_loss": 0.6138982772827148,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07484424859285355,
      "rewards/margins": 0.018103793263435364,
      "rewards/rejected": -0.09294802695512772,
      "sft_loss": 0.7484423518180847,
      "step": 1000
    },
    {
      "epoch": 1.6934801016088061,
      "eval_logits/chosen": -3.0966453552246094,
      "eval_logits/rejected": -3.117032051086426,
      "eval_logps/chosen": -0.7911127805709839,
      "eval_logps/rejected": -0.9985377192497253,
      "eval_loss": 0.8548597693443298,
      "eval_odds_ratio_loss": 0.6374707221984863,
      "eval_rewards/accuracies": 0.5676190257072449,
      "eval_rewards/chosen": -0.07911127805709839,
      "eval_rewards/margins": 0.020742492750287056,
      "eval_rewards/rejected": -0.0998537689447403,
      "eval_runtime": 195.0995,
      "eval_samples_per_second": 5.382,
      "eval_sft_loss": 0.7911127805709839,
      "eval_steps_per_second": 2.691,
      "step": 1000
    },
    {
      "epoch": 1.710414902624894,
      "grad_norm": 2.5681636333465576,
      "learning_rate": 1.9500653531031917e-06,
      "logits/chosen": -3.116891384124756,
      "logits/rejected": -3.1181325912475586,
      "logps/chosen": -0.8085691332817078,
      "logps/rejected": -1.0325000286102295,
      "loss": 0.8768,
      "odds_ratio_loss": 0.6825646162033081,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08085691183805466,
      "rewards/margins": 0.022393101826310158,
      "rewards/rejected": -0.10325001180171967,
      "sft_loss": 0.8085691332817078,
      "step": 1010
    },
    {
      "epoch": 1.7273497036409822,
      "grad_norm": 2.5795137882232666,
      "learning_rate": 1.9068659108055117e-06,
      "logits/chosen": -3.1321749687194824,
      "logits/rejected": -3.155836582183838,
      "logps/chosen": -0.7755477428436279,
      "logps/rejected": -0.9410096406936646,
      "loss": 0.8406,
      "odds_ratio_loss": 0.6507992744445801,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07755477726459503,
      "rewards/margins": 0.016546186059713364,
      "rewards/rejected": -0.0941009670495987,
      "sft_loss": 0.7755477428436279,
      "step": 1020
    },
    {
      "epoch": 1.7442845046570703,
      "grad_norm": 2.360201835632324,
      "learning_rate": 1.863853340307962e-06,
      "logits/chosen": -3.078691005706787,
      "logits/rejected": -3.1044058799743652,
      "logps/chosen": -0.683570146560669,
      "logps/rejected": -0.9945551753044128,
      "loss": 0.7414,
      "odds_ratio_loss": 0.5782071352005005,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06835701316595078,
      "rewards/margins": 0.03109849989414215,
      "rewards/rejected": -0.09945552051067352,
      "sft_loss": 0.683570146560669,
      "step": 1030
    },
    {
      "epoch": 1.7612193056731584,
      "grad_norm": 2.180799961090088,
      "learning_rate": 1.8210411930766019e-06,
      "logits/chosen": -3.0688931941986084,
      "logits/rejected": -3.096926212310791,
      "logps/chosen": -0.7589991688728333,
      "logps/rejected": -0.9845136404037476,
      "loss": 0.8196,
      "odds_ratio_loss": 0.6064754128456116,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07589991390705109,
      "rewards/margins": 0.02255145087838173,
      "rewards/rejected": -0.09845136106014252,
      "sft_loss": 0.7589991688728333,
      "step": 1040
    },
    {
      "epoch": 1.7781541066892466,
      "grad_norm": 2.4373152256011963,
      "learning_rate": 1.7784429574324803e-06,
      "logits/chosen": -3.079857349395752,
      "logits/rejected": -3.1024343967437744,
      "logps/chosen": -0.738146960735321,
      "logps/rejected": -0.9651134610176086,
      "loss": 0.7991,
      "odds_ratio_loss": 0.6093234419822693,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07381470501422882,
      "rewards/margins": 0.022696642205119133,
      "rewards/rejected": -0.0965113490819931,
      "sft_loss": 0.738146960735321,
      "step": 1050
    },
    {
      "epoch": 1.7950889077053345,
      "grad_norm": 1.4168922901153564,
      "learning_rate": 1.7360720543020327e-06,
      "logits/chosen": -3.158327102661133,
      "logits/rejected": -3.1482691764831543,
      "logps/chosen": -0.7429002523422241,
      "logps/rejected": -0.9451411366462708,
      "loss": 0.8021,
      "odds_ratio_loss": 0.5917203426361084,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07429002225399017,
      "rewards/margins": 0.020224085077643394,
      "rewards/rejected": -0.09451410919427872,
      "sft_loss": 0.7429002523422241,
      "step": 1060
    },
    {
      "epoch": 1.8120237087214224,
      "grad_norm": 7.384251594543457,
      "learning_rate": 1.6939418329887042e-06,
      "logits/chosen": -3.1223366260528564,
      "logits/rejected": -3.157349109649658,
      "logps/chosen": -0.7720141410827637,
      "logps/rejected": -1.004219651222229,
      "loss": 0.8344,
      "odds_ratio_loss": 0.623358428478241,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07720141112804413,
      "rewards/margins": 0.023220548406243324,
      "rewards/rejected": -0.1004219651222229,
      "sft_loss": 0.7720141410827637,
      "step": 1070
    },
    {
      "epoch": 1.8289585097375105,
      "grad_norm": 1.976076602935791,
      "learning_rate": 1.6520655669671467e-06,
      "logits/chosen": -3.151508331298828,
      "logits/rejected": -3.174203395843506,
      "logps/chosen": -0.7854975461959839,
      "logps/rejected": -0.9663504362106323,
      "loss": 0.8499,
      "odds_ratio_loss": 0.6441539525985718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07854975759983063,
      "rewards/margins": 0.01808529533445835,
      "rewards/rejected": -0.09663505107164383,
      "sft_loss": 0.7854975461959839,
      "step": 1080
    },
    {
      "epoch": 1.8458933107535986,
      "grad_norm": 2.3342175483703613,
      "learning_rate": 1.610456449701294e-06,
      "logits/chosen": -3.110039234161377,
      "logits/rejected": -3.1346383094787598,
      "logps/chosen": -0.7955976724624634,
      "logps/rejected": -1.0117195844650269,
      "loss": 0.8634,
      "odds_ratio_loss": 0.6780521273612976,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07955978065729141,
      "rewards/margins": 0.02161218598484993,
      "rewards/rejected": -0.10117195546627045,
      "sft_loss": 0.7955976724624634,
      "step": 1090
    },
    {
      "epoch": 1.8628281117696868,
      "grad_norm": 2.040372133255005,
      "learning_rate": 1.5691275904876545e-06,
      "logits/chosen": -3.11810302734375,
      "logits/rejected": -3.0907464027404785,
      "logps/chosen": -0.8171418905258179,
      "logps/rejected": -1.049744963645935,
      "loss": 0.8794,
      "odds_ratio_loss": 0.6221813559532166,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0817141979932785,
      "rewards/margins": 0.02326030097901821,
      "rewards/rejected": -0.10497449338436127,
      "sft_loss": 0.8171418905258179,
      "step": 1100
    },
    {
      "epoch": 1.879762912785775,
      "grad_norm": 1.1363812685012817,
      "learning_rate": 1.5280920103251235e-06,
      "logits/chosen": -3.121904134750366,
      "logits/rejected": -3.1400184631347656,
      "logps/chosen": -0.7571579217910767,
      "logps/rejected": -0.9578372240066528,
      "loss": 0.8213,
      "odds_ratio_loss": 0.6416117548942566,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07571578770875931,
      "rewards/margins": 0.020067930221557617,
      "rewards/rejected": -0.09578372538089752,
      "sft_loss": 0.7571579217910767,
      "step": 1110
    },
    {
      "epoch": 1.8966977138018628,
      "grad_norm": 3.5236151218414307,
      "learning_rate": 1.4873626378126015e-06,
      "logits/chosen": -3.109051465988159,
      "logits/rejected": -3.123108386993408,
      "logps/chosen": -0.7997997403144836,
      "logps/rejected": -1.0338528156280518,
      "loss": 0.8642,
      "odds_ratio_loss": 0.6441462635993958,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07997997850179672,
      "rewards/margins": 0.02340531349182129,
      "rewards/rejected": -0.10338529199361801,
      "sft_loss": 0.7997997403144836,
      "step": 1120
    },
    {
      "epoch": 1.913632514817951,
      "grad_norm": 4.680517673492432,
      "learning_rate": 1.446952305075738e-06,
      "logits/chosen": -3.1131389141082764,
      "logits/rejected": -3.151978015899658,
      "logps/chosen": -0.7748031616210938,
      "logps/rejected": -0.8985496759414673,
      "loss": 0.8402,
      "odds_ratio_loss": 0.6542429327964783,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07748032361268997,
      "rewards/margins": 0.012374645099043846,
      "rewards/rejected": -0.08985497057437897,
      "sft_loss": 0.7748031616210938,
      "step": 1130
    },
    {
      "epoch": 1.9305673158340388,
      "grad_norm": 1.990546703338623,
      "learning_rate": 1.406873743724065e-06,
      "logits/chosen": -3.134593963623047,
      "logits/rejected": -3.088059186935425,
      "logps/chosen": -0.8194819688796997,
      "logps/rejected": -1.0895836353302002,
      "loss": 0.8842,
      "odds_ratio_loss": 0.6472653746604919,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08194819837808609,
      "rewards/margins": 0.02701016701757908,
      "rewards/rejected": -0.10895836353302002,
      "sft_loss": 0.8194819688796997,
      "step": 1140
    },
    {
      "epoch": 1.947502116850127,
      "grad_norm": 2.0966198444366455,
      "learning_rate": 1.3671395808397898e-06,
      "logits/chosen": -3.1003191471099854,
      "logits/rejected": -3.159738779067993,
      "logps/chosen": -0.7502952218055725,
      "logps/rejected": -0.8763904571533203,
      "loss": 0.8142,
      "odds_ratio_loss": 0.6395031213760376,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07502951472997665,
      "rewards/margins": 0.012609531171619892,
      "rewards/rejected": -0.08763904869556427,
      "sft_loss": 0.7502952218055725,
      "step": 1150
    },
    {
      "epoch": 1.964436917866215,
      "grad_norm": 6.19160795211792,
      "learning_rate": 1.3277623349995418e-06,
      "logits/chosen": -3.1115336418151855,
      "logits/rejected": -3.1329808235168457,
      "logps/chosen": -0.7726483941078186,
      "logps/rejected": -0.9365378618240356,
      "loss": 0.8381,
      "odds_ratio_loss": 0.6545372605323792,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07726484537124634,
      "rewards/margins": 0.016388945281505585,
      "rewards/rejected": -0.09365378320217133,
      "sft_loss": 0.7726483941078186,
      "step": 1160
    },
    {
      "epoch": 1.9813717188823032,
      "grad_norm": 2.1759722232818604,
      "learning_rate": 1.2887544123302781e-06,
      "logits/chosen": -3.1343424320220947,
      "logits/rejected": -3.145904064178467,
      "logps/chosen": -0.795665442943573,
      "logps/rejected": -0.9278801679611206,
      "loss": 0.863,
      "odds_ratio_loss": 0.6731692552566528,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07956655323505402,
      "rewards/margins": 0.01322146225720644,
      "rewards/rejected": -0.09278801828622818,
      "sft_loss": 0.795665442943573,
      "step": 1170
    },
    {
      "epoch": 1.9983065198983911,
      "grad_norm": 2.340391159057617,
      "learning_rate": 1.2501281026006393e-06,
      "logits/chosen": -3.113882064819336,
      "logits/rejected": -3.1493611335754395,
      "logps/chosen": -0.7624078989028931,
      "logps/rejected": -0.9082571268081665,
      "loss": 0.8289,
      "odds_ratio_loss": 0.6652835607528687,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07624078541994095,
      "rewards/margins": 0.014584928750991821,
      "rewards/rejected": -0.09082571417093277,
      "sft_loss": 0.7624078989028931,
      "step": 1180
    },
    {
      "epoch": 2.015241320914479,
      "grad_norm": 2.1316022872924805,
      "learning_rate": 1.2118955753489523e-06,
      "logits/chosen": -3.1328041553497314,
      "logits/rejected": -3.119529962539673,
      "logps/chosen": -0.7868901491165161,
      "logps/rejected": -0.9673662185668945,
      "loss": 0.8514,
      "odds_ratio_loss": 0.6448970437049866,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07868902385234833,
      "rewards/margins": 0.018047606572508812,
      "rewards/rejected": -0.09673662483692169,
      "sft_loss": 0.7868901491165161,
      "step": 1190
    },
    {
      "epoch": 2.032176121930567,
      "grad_norm": 2.219909906387329,
      "learning_rate": 1.1740688760491189e-06,
      "logits/chosen": -3.1113944053649902,
      "logits/rejected": -3.1537253856658936,
      "logps/chosen": -0.7722674608230591,
      "logps/rejected": -1.0175268650054932,
      "loss": 0.8299,
      "odds_ratio_loss": 0.5762220621109009,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07722674310207367,
      "rewards/margins": 0.02452593669295311,
      "rewards/rejected": -0.10175268352031708,
      "sft_loss": 0.7722674608230591,
      "step": 1200
    },
    {
      "epoch": 2.0491109229466553,
      "grad_norm": 6.625383377075195,
      "learning_rate": 1.1366599223155847e-06,
      "logits/chosen": -3.092228651046753,
      "logits/rejected": -3.137305498123169,
      "logps/chosen": -0.7458280324935913,
      "logps/rejected": -1.1138603687286377,
      "loss": 0.8056,
      "odds_ratio_loss": 0.5978988409042358,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07458280026912689,
      "rewards/margins": 0.0368032269179821,
      "rewards/rejected": -0.1113860234618187,
      "sft_loss": 0.7458280324935913,
      "step": 1210
    },
    {
      "epoch": 2.0660457239627434,
      "grad_norm": 3.234821081161499,
      "learning_rate": 1.0996805001486067e-06,
      "logits/chosen": -3.109764575958252,
      "logits/rejected": -3.1344008445739746,
      "logps/chosen": -0.7534822225570679,
      "logps/rejected": -0.9295659065246582,
      "loss": 0.8161,
      "odds_ratio_loss": 0.625883162021637,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07534822076559067,
      "rewards/margins": 0.017608370631933212,
      "rewards/rejected": -0.09295658767223358,
      "sft_loss": 0.7534822225570679,
      "step": 1220
    },
    {
      "epoch": 2.0829805249788316,
      "grad_norm": 3.5898754596710205,
      "learning_rate": 1.0631422602209608e-06,
      "logits/chosen": -3.1376945972442627,
      "logits/rejected": -3.1564555168151855,
      "logps/chosen": -0.7927883863449097,
      "logps/rejected": -0.9650642275810242,
      "loss": 0.8569,
      "odds_ratio_loss": 0.6410055160522461,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0792788416147232,
      "rewards/margins": 0.017227591946721077,
      "rewards/rejected": -0.09650643169879913,
      "sft_loss": 0.7927883863449097,
      "step": 1230
    },
    {
      "epoch": 2.0999153259949197,
      "grad_norm": 1.6415690183639526,
      "learning_rate": 1.027056714207319e-06,
      "logits/chosen": -3.1497724056243896,
      "logits/rejected": -3.1502339839935303,
      "logps/chosen": -0.8299382925033569,
      "logps/rejected": -1.0621525049209595,
      "loss": 0.8939,
      "odds_ratio_loss": 0.6398864984512329,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08299383521080017,
      "rewards/margins": 0.023221401497721672,
      "rewards/rejected": -0.10621523857116699,
      "sft_loss": 0.8299382925033569,
      "step": 1240
    },
    {
      "epoch": 2.116850127011008,
      "grad_norm": 2.1568691730499268,
      "learning_rate": 9.914352311573838e-07,
      "logits/chosen": -3.1183249950408936,
      "logits/rejected": -3.130068302154541,
      "logps/chosen": -0.680639386177063,
      "logps/rejected": -0.9202540516853333,
      "loss": 0.7394,
      "odds_ratio_loss": 0.5878284573554993,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06806393712759018,
      "rewards/margins": 0.02396145835518837,
      "rewards/rejected": -0.09202539175748825,
      "sft_loss": 0.680639386177063,
      "step": 1250
    },
    {
      "epoch": 2.1337849280270955,
      "grad_norm": 1.6225751638412476,
      "learning_rate": 9.562890339139877e-07,
      "logits/chosen": -3.0915961265563965,
      "logits/rejected": -3.1580090522766113,
      "logps/chosen": -0.7393635511398315,
      "logps/rejected": -0.9581485986709595,
      "loss": 0.8032,
      "odds_ratio_loss": 0.6382402181625366,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07393636554479599,
      "rewards/margins": 0.021878493949770927,
      "rewards/rejected": -0.09581486135721207,
      "sft_loss": 0.7393635511398315,
      "step": 1260
    },
    {
      "epoch": 2.1507197290431836,
      "grad_norm": 1.8676691055297852,
      "learning_rate": 9.216291955772374e-07,
      "logits/chosen": -3.0996224880218506,
      "logits/rejected": -3.153738021850586,
      "logps/chosen": -0.7607396841049194,
      "logps/rejected": -0.9290043115615845,
      "loss": 0.8238,
      "odds_ratio_loss": 0.6303194761276245,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07607396692037582,
      "rewards/margins": 0.016826456412672997,
      "rewards/rejected": -0.09290042519569397,
      "sft_loss": 0.7607396841049194,
      "step": 1270
    },
    {
      "epoch": 2.167654530059272,
      "grad_norm": 2.096705198287964,
      "learning_rate": 8.874666360158457e-07,
      "logits/chosen": -3.082366466522217,
      "logits/rejected": -3.0917139053344727,
      "logps/chosen": -0.7106717824935913,
      "logps/rejected": -0.9525697827339172,
      "loss": 0.774,
      "odds_ratio_loss": 0.6332431435585022,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07106717675924301,
      "rewards/margins": 0.024189796298742294,
      "rewards/rejected": -0.0952569767832756,
      "sft_loss": 0.7106717824935913,
      "step": 1280
    },
    {
      "epoch": 2.18458933107536,
      "grad_norm": 2.435957431793213,
      "learning_rate": 8.538121184267315e-07,
      "logits/chosen": -3.1009063720703125,
      "logits/rejected": -3.121425151824951,
      "logps/chosen": -0.6681427955627441,
      "logps/rejected": -0.9359544515609741,
      "loss": 0.7277,
      "odds_ratio_loss": 0.595305323600769,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06681428104639053,
      "rewards/margins": 0.026781165972352028,
      "rewards/rejected": -0.09359544515609741,
      "sft_loss": 0.6681427955627441,
      "step": 1290
    },
    {
      "epoch": 2.201524132091448,
      "grad_norm": 2.3414995670318604,
      "learning_rate": 8.206762459439907e-07,
      "logits/chosen": -3.1007769107818604,
      "logits/rejected": -3.142437696456909,
      "logps/chosen": -0.7944619059562683,
      "logps/rejected": -0.9959940910339355,
      "loss": 0.8604,
      "odds_ratio_loss": 0.659730076789856,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07944619655609131,
      "rewards/margins": 0.020153220742940903,
      "rewards/rejected": -0.09959942102432251,
      "sft_loss": 0.7944619059562683,
      "step": 1300
    },
    {
      "epoch": 2.218458933107536,
      "grad_norm": 2.9054501056671143,
      "learning_rate": 7.880694582982898e-07,
      "logits/chosen": -3.165544271469116,
      "logits/rejected": -3.1803879737854004,
      "logps/chosen": -0.8287284970283508,
      "logps/rejected": -1.0228365659713745,
      "loss": 0.8926,
      "odds_ratio_loss": 0.6389774084091187,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08287284523248672,
      "rewards/margins": 0.019410807639360428,
      "rewards/rejected": -0.10228364169597626,
      "sft_loss": 0.8287284970283508,
      "step": 1310
    },
    {
      "epoch": 2.235393734123624,
      "grad_norm": 3.0612242221832275,
      "learning_rate": 7.560020285277401e-07,
      "logits/chosen": -3.0883891582489014,
      "logits/rejected": -3.142484426498413,
      "logps/chosen": -0.7654698491096497,
      "logps/rejected": -0.8691753149032593,
      "loss": 0.8363,
      "odds_ratio_loss": 0.708137035369873,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07654698193073273,
      "rewards/margins": 0.010370554402470589,
      "rewards/rejected": -0.08691753447055817,
      "sft_loss": 0.7654698491096497,
      "step": 1320
    },
    {
      "epoch": 2.252328535139712,
      "grad_norm": 3.074390172958374,
      "learning_rate": 7.244840597412956e-07,
      "logits/chosen": -3.0840373039245605,
      "logits/rejected": -3.113865613937378,
      "logps/chosen": -0.7826686501502991,
      "logps/rejected": -0.8837703466415405,
      "loss": 0.8534,
      "odds_ratio_loss": 0.7077327370643616,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07826686650514603,
      "rewards/margins": 0.010110177099704742,
      "rewards/rejected": -0.08837703615427017,
      "sft_loss": 0.7826686501502991,
      "step": 1330
    },
    {
      "epoch": 2.2692633361558,
      "grad_norm": 6.461457252502441,
      "learning_rate": 6.935254819356796e-07,
      "logits/chosen": -3.1297388076782227,
      "logits/rejected": -3.1470894813537598,
      "logps/chosen": -0.7817297577857971,
      "logps/rejected": -0.9597527384757996,
      "loss": 0.8464,
      "odds_ratio_loss": 0.6471723318099976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07817298173904419,
      "rewards/margins": 0.017802301794290543,
      "rewards/rejected": -0.09597527980804443,
      "sft_loss": 0.7817297577857971,
      "step": 1340
    },
    {
      "epoch": 2.2861981371718882,
      "grad_norm": 1.7192656993865967,
      "learning_rate": 6.631360488668662e-07,
      "logits/chosen": -3.138521194458008,
      "logits/rejected": -3.150005578994751,
      "logps/chosen": -0.7190467715263367,
      "logps/rejected": -0.9147864580154419,
      "loss": 0.7824,
      "odds_ratio_loss": 0.6330953240394592,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07190467417240143,
      "rewards/margins": 0.019573967903852463,
      "rewards/rejected": -0.09147863835096359,
      "sft_loss": 0.7190467715263367,
      "step": 1350
    },
    {
      "epoch": 2.3031329381879764,
      "grad_norm": 2.849228620529175,
      "learning_rate": 6.333253349770672e-07,
      "logits/chosen": -3.1443207263946533,
      "logits/rejected": -3.1502537727355957,
      "logps/chosen": -0.8002703785896301,
      "logps/rejected": -0.9851358532905579,
      "loss": 0.8656,
      "odds_ratio_loss": 0.6537164449691772,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0800270363688469,
      "rewards/margins": 0.018486548215150833,
      "rewards/rejected": -0.09851358830928802,
      "sft_loss": 0.8002703785896301,
      "step": 1360
    },
    {
      "epoch": 2.3200677392040645,
      "grad_norm": 1.3985761404037476,
      "learning_rate": 6.041027323782364e-07,
      "logits/chosen": -3.136557102203369,
      "logits/rejected": -3.1552491188049316,
      "logps/chosen": -0.7381452918052673,
      "logps/rejected": -0.9497495889663696,
      "loss": 0.7993,
      "odds_ratio_loss": 0.611694872379303,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07381454110145569,
      "rewards/margins": 0.021160420030355453,
      "rewards/rejected": -0.09497495740652084,
      "sft_loss": 0.7381452918052673,
      "step": 1370
    },
    {
      "epoch": 2.337002540220152,
      "grad_norm": 2.8737893104553223,
      "learning_rate": 5.754774478929969e-07,
      "logits/chosen": -3.1531028747558594,
      "logits/rejected": -3.1830787658691406,
      "logps/chosen": -0.7512658834457397,
      "logps/rejected": -0.9563377499580383,
      "loss": 0.81,
      "odds_ratio_loss": 0.5872438549995422,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07512658089399338,
      "rewards/margins": 0.020507195964455605,
      "rewards/rejected": -0.09563378244638443,
      "sft_loss": 0.7512658834457397,
      "step": 1380
    },
    {
      "epoch": 2.3539373412362403,
      "grad_norm": 3.9828193187713623,
      "learning_rate": 5.474585001539634e-07,
      "logits/chosen": -3.159769296646118,
      "logits/rejected": -3.1827354431152344,
      "logps/chosen": -0.716742753982544,
      "logps/rejected": -0.9266722798347473,
      "loss": 0.7742,
      "odds_ratio_loss": 0.5742050409317017,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07167427241802216,
      "rewards/margins": 0.020992957055568695,
      "rewards/rejected": -0.09266723692417145,
      "sft_loss": 0.716742753982544,
      "step": 1390
    },
    {
      "epoch": 2.3708721422523285,
      "grad_norm": 1.6741605997085571,
      "learning_rate": 5.200547167623424e-07,
      "logits/chosen": -3.172938823699951,
      "logits/rejected": -3.1454200744628906,
      "logps/chosen": -0.7799001932144165,
      "logps/rejected": -1.040466070175171,
      "loss": 0.8399,
      "odds_ratio_loss": 0.5997284650802612,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07799001783132553,
      "rewards/margins": 0.026056593284010887,
      "rewards/rejected": -0.10404660552740097,
      "sft_loss": 0.7799001932144165,
      "step": 1400
    },
    {
      "epoch": 2.3878069432684166,
      "grad_norm": 1.3470762968063354,
      "learning_rate": 4.932747315067271e-07,
      "logits/chosen": -3.15238356590271,
      "logits/rejected": -3.1575927734375,
      "logps/chosen": -0.7618133425712585,
      "logps/rejected": -1.033060073852539,
      "loss": 0.8193,
      "odds_ratio_loss": 0.574641764163971,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0761813372373581,
      "rewards/margins": 0.027124667540192604,
      "rewards/rejected": -0.10330601036548615,
      "sft_loss": 0.7618133425712585,
      "step": 1410
    },
    {
      "epoch": 2.4047417442845047,
      "grad_norm": 4.0449371337890625,
      "learning_rate": 4.6712698164294553e-07,
      "logits/chosen": -3.1169888973236084,
      "logits/rejected": -3.131412982940674,
      "logps/chosen": -0.7451744079589844,
      "logps/rejected": -0.9645574688911438,
      "loss": 0.8029,
      "odds_ratio_loss": 0.5777136087417603,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07451744377613068,
      "rewards/margins": 0.021938303485512733,
      "rewards/rejected": -0.09645574539899826,
      "sft_loss": 0.7451744079589844,
      "step": 1420
    },
    {
      "epoch": 2.421676545300593,
      "grad_norm": 1.97295343875885,
      "learning_rate": 4.41619705235842e-07,
      "logits/chosen": -3.1363072395324707,
      "logits/rejected": -3.1507456302642822,
      "logps/chosen": -0.7545329332351685,
      "logps/rejected": -1.082715392112732,
      "loss": 0.8124,
      "odds_ratio_loss": 0.5782482624053955,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07545328885316849,
      "rewards/margins": 0.0328182615339756,
      "rewards/rejected": -0.10827155411243439,
      "sft_loss": 0.7545329332351685,
      "step": 1430
    },
    {
      "epoch": 2.438611346316681,
      "grad_norm": 2.880293846130371,
      "learning_rate": 4.167609385637961e-07,
      "logits/chosen": -3.182002544403076,
      "logits/rejected": -3.165118455886841,
      "logps/chosen": -0.7962235808372498,
      "logps/rejected": -0.975197970867157,
      "loss": 0.8596,
      "odds_ratio_loss": 0.634021520614624,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07962235808372498,
      "rewards/margins": 0.017897438257932663,
      "rewards/rejected": -0.09751980006694794,
      "sft_loss": 0.7962235808372498,
      "step": 1440
    },
    {
      "epoch": 2.4555461473327687,
      "grad_norm": 2.545621871948242,
      "learning_rate": 3.9255851358683567e-07,
      "logits/chosen": -3.1265337467193604,
      "logits/rejected": -3.140825033187866,
      "logps/chosen": -0.7502083778381348,
      "logps/rejected": -0.9492311477661133,
      "loss": 0.817,
      "odds_ratio_loss": 0.6676734685897827,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07502084225416183,
      "rewards/margins": 0.019902262836694717,
      "rewards/rejected": -0.09492311626672745,
      "sft_loss": 0.7502083778381348,
      "step": 1450
    },
    {
      "epoch": 2.472480948348857,
      "grad_norm": 2.91165828704834,
      "learning_rate": 3.690200554791082e-07,
      "logits/chosen": -3.093761444091797,
      "logits/rejected": -3.0926525592803955,
      "logps/chosen": -0.7423545122146606,
      "logps/rejected": -0.9597676992416382,
      "loss": 0.8001,
      "odds_ratio_loss": 0.5777243375778198,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0742354542016983,
      "rewards/margins": 0.02174132689833641,
      "rewards/rejected": -0.09597676992416382,
      "sft_loss": 0.7423545122146606,
      "step": 1460
    },
    {
      "epoch": 2.489415749364945,
      "grad_norm": 5.057043075561523,
      "learning_rate": 3.461529802265079e-07,
      "logits/chosen": -3.153536796569824,
      "logits/rejected": -3.16294264793396,
      "logps/chosen": -0.7584110498428345,
      "logps/rejected": -0.9484812617301941,
      "loss": 0.8201,
      "odds_ratio_loss": 0.6163991093635559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07584110647439957,
      "rewards/margins": 0.01900702901184559,
      "rewards/rejected": -0.0948481336236,
      "sft_loss": 0.7584110498428345,
      "step": 1470
    },
    {
      "epoch": 2.506350550381033,
      "grad_norm": 2.0298070907592773,
      "learning_rate": 3.2396449229020883e-07,
      "logits/chosen": -3.18571400642395,
      "logits/rejected": -3.1676132678985596,
      "logps/chosen": -0.7937291264533997,
      "logps/rejected": -0.935733437538147,
      "loss": 0.8599,
      "odds_ratio_loss": 0.6614922881126404,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07937291264533997,
      "rewards/margins": 0.014200428500771523,
      "rewards/rejected": -0.09357334673404694,
      "sft_loss": 0.7937291264533997,
      "step": 1480
    },
    {
      "epoch": 2.523285351397121,
      "grad_norm": 4.000706195831299,
      "learning_rate": 3.024615823368371e-07,
      "logits/chosen": -3.1206421852111816,
      "logits/rejected": -3.152639150619507,
      "logps/chosen": -0.7629178166389465,
      "logps/rejected": -0.9828959703445435,
      "loss": 0.8252,
      "odds_ratio_loss": 0.622687041759491,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07629178464412689,
      "rewards/margins": 0.021997807547450066,
      "rewards/rejected": -0.09828958660364151,
      "sft_loss": 0.7629178166389465,
      "step": 1490
    },
    {
      "epoch": 2.5402201524132093,
      "grad_norm": 2.449856758117676,
      "learning_rate": 2.8165102503600716e-07,
      "logits/chosen": -3.095520496368408,
      "logits/rejected": -3.1078646183013916,
      "logps/chosen": -0.7517341375350952,
      "logps/rejected": -0.9984272718429565,
      "loss": 0.8135,
      "odds_ratio_loss": 0.6172657012939453,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07517342269420624,
      "rewards/margins": 0.024669310078024864,
      "rewards/rejected": -0.09984272718429565,
      "sft_loss": 0.7517341375350952,
      "step": 1500
    },
    {
      "epoch": 2.5402201524132093,
      "eval_logits/chosen": -3.130527973175049,
      "eval_logits/rejected": -3.1507296562194824,
      "eval_logps/chosen": -0.7867480516433716,
      "eval_logps/rejected": -0.9955620169639587,
      "eval_loss": 0.8505691885948181,
      "eval_odds_ratio_loss": 0.638211727142334,
      "eval_rewards/accuracies": 0.5723809599876404,
      "eval_rewards/chosen": -0.0786748081445694,
      "eval_rewards/margins": 0.020881392061710358,
      "eval_rewards/rejected": -0.09955620020627975,
      "eval_runtime": 194.4899,
      "eval_samples_per_second": 5.399,
      "eval_sft_loss": 0.7867480516433716,
      "eval_steps_per_second": 2.699,
      "step": 1500
    },
    {
      "epoch": 2.557154953429297,
      "grad_norm": 1.5426675081253052,
      "learning_rate": 2.615393769259039e-07,
      "logits/chosen": -3.1200222969055176,
      "logits/rejected": -3.151517152786255,
      "logps/chosen": -0.845578670501709,
      "logps/rejected": -0.9381099939346313,
      "loss": 0.9194,
      "odds_ratio_loss": 0.7380752563476562,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.08455787599086761,
      "rewards/margins": 0.00925312377512455,
      "rewards/rejected": -0.09381099790334702,
      "sft_loss": 0.845578670501709,
      "step": 1510
    },
    {
      "epoch": 2.574089754445385,
      "grad_norm": 4.487434387207031,
      "learning_rate": 2.421329743475917e-07,
      "logits/chosen": -3.1358139514923096,
      "logits/rejected": -3.157161235809326,
      "logps/chosen": -0.7332046627998352,
      "logps/rejected": -0.9182003140449524,
      "loss": 0.7981,
      "odds_ratio_loss": 0.6488706469535828,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07332046329975128,
      "rewards/margins": 0.018499553203582764,
      "rewards/rejected": -0.09182002395391464,
      "sft_loss": 0.7332046627998352,
      "step": 1520
    },
    {
      "epoch": 2.5910245554614733,
      "grad_norm": 2.3630266189575195,
      "learning_rate": 2.234379314486973e-07,
      "logits/chosen": -3.1165411472320557,
      "logits/rejected": -3.1539194583892822,
      "logps/chosen": -0.8104802370071411,
      "logps/rejected": -0.9215306043624878,
      "loss": 0.8756,
      "odds_ratio_loss": 0.6507223844528198,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08104802668094635,
      "rewards/margins": 0.011105035431683064,
      "rewards/rejected": -0.09215305000543594,
      "sft_loss": 0.8104802370071411,
      "step": 1530
    },
    {
      "epoch": 2.6079593564775614,
      "grad_norm": 1.243202805519104,
      "learning_rate": 2.0546013825709783e-07,
      "logits/chosen": -3.1084282398223877,
      "logits/rejected": -3.129692792892456,
      "logps/chosen": -0.7565353512763977,
      "logps/rejected": -1.0974582433700562,
      "loss": 0.8149,
      "odds_ratio_loss": 0.5834510326385498,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07565353065729141,
      "rewards/margins": 0.034092292189598083,
      "rewards/rejected": -0.1097458228468895,
      "sft_loss": 0.7565353512763977,
      "step": 1540
    },
    {
      "epoch": 2.6248941574936495,
      "grad_norm": 1.8778706789016724,
      "learning_rate": 1.88205258825217e-07,
      "logits/chosen": -3.0997517108917236,
      "logits/rejected": -3.0937392711639404,
      "logps/chosen": -0.672328770160675,
      "logps/rejected": -0.9460701942443848,
      "loss": 0.7295,
      "odds_ratio_loss": 0.5718902349472046,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06723286956548691,
      "rewards/margins": 0.02737414464354515,
      "rewards/rejected": -0.09460701793432236,
      "sft_loss": 0.672328770160675,
      "step": 1550
    },
    {
      "epoch": 2.6418289585097376,
      "grad_norm": 3.3697926998138428,
      "learning_rate": 1.7167872944552245e-07,
      "logits/chosen": -3.0951905250549316,
      "logits/rejected": -3.143221378326416,
      "logps/chosen": -0.6983746290206909,
      "logps/rejected": -0.9046236276626587,
      "loss": 0.7572,
      "odds_ratio_loss": 0.5877906084060669,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06983745843172073,
      "rewards/margins": 0.020624909549951553,
      "rewards/rejected": -0.09046236425638199,
      "sft_loss": 0.6983746290206909,
      "step": 1560
    },
    {
      "epoch": 2.6587637595258258,
      "grad_norm": 2.3217389583587646,
      "learning_rate": 1.5588575693777142e-07,
      "logits/chosen": -3.1411678791046143,
      "logits/rejected": -3.1680665016174316,
      "logps/chosen": -0.7495251297950745,
      "logps/rejected": -0.8962133526802063,
      "loss": 0.8135,
      "odds_ratio_loss": 0.6393758058547974,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07495252043008804,
      "rewards/margins": 0.01466882973909378,
      "rewards/rejected": -0.08962134271860123,
      "sft_loss": 0.7495251297950745,
      "step": 1570
    },
    {
      "epoch": 2.675698560541914,
      "grad_norm": 2.695345401763916,
      "learning_rate": 1.4083131700856428e-07,
      "logits/chosen": -3.093736410140991,
      "logits/rejected": -3.154810905456543,
      "logps/chosen": -0.7956855893135071,
      "logps/rejected": -0.9356697201728821,
      "loss": 0.8605,
      "odds_ratio_loss": 0.6480044722557068,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07956856489181519,
      "rewards/margins": 0.013998406007885933,
      "rewards/rejected": -0.09356696903705597,
      "sft_loss": 0.7956855893135071,
      "step": 1580
    },
    {
      "epoch": 2.6926333615580016,
      "grad_norm": 4.358800888061523,
      "learning_rate": 1.2652015268370315e-07,
      "logits/chosen": -3.150224208831787,
      "logits/rejected": -3.183022975921631,
      "logps/chosen": -0.7435088753700256,
      "logps/rejected": -0.9757458567619324,
      "loss": 0.8044,
      "odds_ratio_loss": 0.6088230013847351,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07435088604688644,
      "rewards/margins": 0.02322370745241642,
      "rewards/rejected": -0.09757460653781891,
      "sft_loss": 0.7435088753700256,
      "step": 1590
    },
    {
      "epoch": 2.7095681625740897,
      "grad_norm": 1.9499818086624146,
      "learning_rate": 1.1295677281386502e-07,
      "logits/chosen": -3.1664371490478516,
      "logits/rejected": -3.1615631580352783,
      "logps/chosen": -0.8278031349182129,
      "logps/rejected": -1.062387228012085,
      "loss": 0.8914,
      "odds_ratio_loss": 0.6362180113792419,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08278031647205353,
      "rewards/margins": 0.02345840446650982,
      "rewards/rejected": -0.1062387228012085,
      "sft_loss": 0.8278031349182129,
      "step": 1600
    },
    {
      "epoch": 2.726502963590178,
      "grad_norm": 1.277694821357727,
      "learning_rate": 1.0014545065404973e-07,
      "logits/chosen": -3.12839674949646,
      "logits/rejected": -3.1691813468933105,
      "logps/chosen": -0.8073375821113586,
      "logps/rejected": -1.0353208780288696,
      "loss": 0.8777,
      "odds_ratio_loss": 0.7032862305641174,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0807337611913681,
      "rewards/margins": 0.022798333317041397,
      "rewards/rejected": -0.1035320907831192,
      "sft_loss": 0.8073375821113586,
      "step": 1610
    },
    {
      "epoch": 2.743437764606266,
      "grad_norm": 1.525190830230713,
      "learning_rate": 8.809022251725502e-08,
      "logits/chosen": -3.1842474937438965,
      "logits/rejected": -3.156172752380371,
      "logps/chosen": -0.7726050615310669,
      "logps/rejected": -1.0311490297317505,
      "loss": 0.8348,
      "odds_ratio_loss": 0.6221681833267212,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07726050913333893,
      "rewards/margins": 0.02585439942777157,
      "rewards/rejected": -0.10311490297317505,
      "sft_loss": 0.7726050615310669,
      "step": 1620
    },
    {
      "epoch": 2.7603725656223537,
      "grad_norm": 2.015005111694336,
      "learning_rate": 7.679488650280509e-08,
      "logits/chosen": -3.192863702774048,
      "logits/rejected": -3.213048219680786,
      "logps/chosen": -0.7830113172531128,
      "logps/rejected": -0.9780160188674927,
      "loss": 0.8429,
      "odds_ratio_loss": 0.5990433096885681,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07830111682415009,
      "rewards/margins": 0.019500473514199257,
      "rewards/rejected": -0.09780160337686539,
      "sft_loss": 0.7830113172531128,
      "step": 1630
    },
    {
      "epoch": 2.777307366638442,
      "grad_norm": 2.5389175415039062,
      "learning_rate": 6.626300129972563e-08,
      "logits/chosen": -3.122981071472168,
      "logits/rejected": -3.214785099029541,
      "logps/chosen": -0.7296528220176697,
      "logps/rejected": -0.955167293548584,
      "loss": 0.792,
      "odds_ratio_loss": 0.6230596303939819,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07296527922153473,
      "rewards/margins": 0.02255145087838173,
      "rewards/rejected": -0.09551674872636795,
      "sft_loss": 0.7296528220176697,
      "step": 1640
    },
    {
      "epoch": 2.79424216765453,
      "grad_norm": 1.7503160238265991,
      "learning_rate": 5.649788506555065e-08,
      "logits/chosen": -3.115635871887207,
      "logits/rejected": -3.166555881500244,
      "logps/chosen": -0.7383006811141968,
      "logps/rejected": -0.9469397664070129,
      "loss": 0.797,
      "odds_ratio_loss": 0.5870878100395203,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07383007556200027,
      "rewards/margins": 0.020863894373178482,
      "rewards/rejected": -0.09469397366046906,
      "sft_loss": 0.7383006811141968,
      "step": 1650
    },
    {
      "epoch": 2.811176968670618,
      "grad_norm": 5.490494728088379,
      "learning_rate": 4.7502614380908474e-08,
      "logits/chosen": -3.1543877124786377,
      "logits/rejected": -3.134704113006592,
      "logps/chosen": -0.7694907188415527,
      "logps/rejected": -0.9553998112678528,
      "loss": 0.8333,
      "odds_ratio_loss": 0.6379188895225525,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07694907486438751,
      "rewards/margins": 0.018590910360217094,
      "rewards/rejected": -0.09553998708724976,
      "sft_loss": 0.7694907188415527,
      "step": 1660
    },
    {
      "epoch": 2.828111769686706,
      "grad_norm": 3.784432888031006,
      "learning_rate": 3.9280023280222066e-08,
      "logits/chosen": -3.1404221057891846,
      "logits/rejected": -3.1622607707977295,
      "logps/chosen": -0.757176399230957,
      "logps/rejected": -0.9332196116447449,
      "loss": 0.8222,
      "odds_ratio_loss": 0.6507169008255005,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07571764290332794,
      "rewards/margins": 0.017604324966669083,
      "rewards/rejected": -0.09332196414470673,
      "sft_loss": 0.757176399230957,
      "step": 1670
    },
    {
      "epoch": 2.8450465707027943,
      "grad_norm": 2.088921546936035,
      "learning_rate": 3.1832702358818855e-08,
      "logits/chosen": -3.116088390350342,
      "logits/rejected": -3.1639227867126465,
      "logps/chosen": -0.8132543563842773,
      "logps/rejected": -1.068440556526184,
      "loss": 0.8739,
      "odds_ratio_loss": 0.6062491536140442,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08132544159889221,
      "rewards/margins": 0.025518611073493958,
      "rewards/rejected": -0.10684405267238617,
      "sft_loss": 0.8132543563842773,
      "step": 1680
    },
    {
      "epoch": 2.8619813717188824,
      "grad_norm": 2.5282764434814453,
      "learning_rate": 2.5162997956746647e-08,
      "logits/chosen": -3.1500442028045654,
      "logits/rejected": -3.171433210372925,
      "logps/chosen": -0.7479134202003479,
      "logps/rejected": -0.9974180459976196,
      "loss": 0.8076,
      "odds_ratio_loss": 0.596734344959259,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0747913345694542,
      "rewards/margins": 0.02495047077536583,
      "rewards/rejected": -0.09974180907011032,
      "sft_loss": 0.7479134202003479,
      "step": 1690
    },
    {
      "epoch": 2.8789161727349706,
      "grad_norm": 3.469899892807007,
      "learning_rate": 1.9273011419536914e-08,
      "logits/chosen": -3.140916585922241,
      "logits/rejected": -3.16453218460083,
      "logps/chosen": -0.7706011533737183,
      "logps/rejected": -0.9274336695671082,
      "loss": 0.8378,
      "odds_ratio_loss": 0.6717150211334229,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07706011831760406,
      "rewards/margins": 0.0156832467764616,
      "rewards/rejected": -0.09274337440729141,
      "sft_loss": 0.7706011533737183,
      "step": 1700
    },
    {
      "epoch": 2.8958509737510583,
      "grad_norm": 1.3444654941558838,
      "learning_rate": 1.4164598436159083e-08,
      "logits/chosen": -3.1658730506896973,
      "logits/rejected": -3.1828174591064453,
      "logps/chosen": -0.7568970322608948,
      "logps/rejected": -0.9151817560195923,
      "loss": 0.82,
      "odds_ratio_loss": 0.630837082862854,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07568971067667007,
      "rewards/margins": 0.01582847535610199,
      "rewards/rejected": -0.09151817858219147,
      "sft_loss": 0.7568970322608948,
      "step": 1710
    },
    {
      "epoch": 2.9127857747671464,
      "grad_norm": 2.7227256298065186,
      "learning_rate": 9.839368454371556e-09,
      "logits/chosen": -3.1114795207977295,
      "logits/rejected": -3.11403751373291,
      "logps/chosen": -0.7409245371818542,
      "logps/rejected": -1.0069682598114014,
      "loss": 0.8046,
      "odds_ratio_loss": 0.6370644569396973,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07409246265888214,
      "rewards/margins": 0.026604369282722473,
      "rewards/rejected": -0.10069682449102402,
      "sft_loss": 0.7409245371818542,
      "step": 1720
    },
    {
      "epoch": 2.9297205757832345,
      "grad_norm": 1.9127988815307617,
      "learning_rate": 6.298684173650649e-09,
      "logits/chosen": -3.0886847972869873,
      "logits/rejected": -3.123133897781372,
      "logps/chosen": -0.7336040735244751,
      "logps/rejected": -1.0557795763015747,
      "loss": 0.7977,
      "odds_ratio_loss": 0.640912652015686,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07336040586233139,
      "rewards/margins": 0.03221754729747772,
      "rewards/rejected": -0.10557796061038971,
      "sft_loss": 0.7336040735244751,
      "step": 1730
    },
    {
      "epoch": 2.9466553767993227,
      "grad_norm": 4.324622631072998,
      "learning_rate": 3.543661115860686e-09,
      "logits/chosen": -3.103651523590088,
      "logits/rejected": -3.136838674545288,
      "logps/chosen": -0.790323793888092,
      "logps/rejected": -1.0052763223648071,
      "loss": 0.8552,
      "odds_ratio_loss": 0.6486952900886536,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07903237640857697,
      "rewards/margins": 0.021495262160897255,
      "rewards/rejected": -0.10052764415740967,
      "sft_loss": 0.790323793888092,
      "step": 1740
    },
    {
      "epoch": 2.963590177815411,
      "grad_norm": 2.1066298484802246,
      "learning_rate": 1.575167273800693e-09,
      "logits/chosen": -3.1322402954101562,
      "logits/rejected": -3.141758680343628,
      "logps/chosen": -0.7295509576797485,
      "logps/rejected": -0.8775620460510254,
      "loss": 0.7914,
      "odds_ratio_loss": 0.6189672350883484,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07295509427785873,
      "rewards/margins": 0.01480111014097929,
      "rewards/rejected": -0.0877562090754509,
      "sft_loss": 0.7295509576797485,
      "step": 1750
    },
    {
      "epoch": 2.9805249788314985,
      "grad_norm": 2.692870855331421,
      "learning_rate": 3.9382283773564676e-10,
      "logits/chosen": -3.153014659881592,
      "logits/rejected": -3.1698267459869385,
      "logps/chosen": -0.8286467790603638,
      "logps/rejected": -1.0176646709442139,
      "loss": 0.8971,
      "odds_ratio_loss": 0.6844185590744019,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0828646719455719,
      "rewards/margins": 0.018901783972978592,
      "rewards/rejected": -0.10176645219326019,
      "sft_loss": 0.8286467790603638,
      "step": 1760
    },
    {
      "epoch": 2.9974597798475866,
      "grad_norm": 11.2490234375,
      "learning_rate": 0.0,
      "logits/chosen": -3.122014284133911,
      "logits/rejected": -3.142989158630371,
      "logps/chosen": -0.8531309962272644,
      "logps/rejected": -1.0983434915542603,
      "loss": 0.9206,
      "odds_ratio_loss": 0.6742203235626221,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0853130966424942,
      "rewards/margins": 0.024521255865693092,
      "rewards/rejected": -0.10983435064554214,
      "sft_loss": 0.8531309962272644,
      "step": 1770
    },
    {
      "epoch": 2.9974597798475866,
      "step": 1770,
      "total_flos": 2.0399855839629804e+18,
      "train_loss": 0.8663494454938813,
      "train_runtime": 17638.0542,
      "train_samples_per_second": 1.607,
      "train_steps_per_second": 0.1
    }
  ],
  "logging_steps": 10,
  "max_steps": 1770,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 2.0399855839629804e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}