{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9973828840617638,
  "eval_steps": 100,
  "global_step": 954,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002093692750588851,
      "grad_norm": 37.29612828590162,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -1.5181711912155151,
      "logits/rejected": -1.4237494468688965,
      "logps/chosen": -331.7369384765625,
      "logps/rejected": -353.880859375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 34.09183299219303,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -1.4457752704620361,
      "logits/rejected": -1.4434000253677368,
      "logps/chosen": -409.06182861328125,
      "logps/rejected": -344.4476318359375,
      "loss": 0.7192,
      "rewards/accuracies": 0.3611111044883728,
      "rewards/chosen": -0.012955614365637302,
      "rewards/margins": 0.009454314596951008,
      "rewards/rejected": -0.02240992709994316,
      "step": 10
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 33.674655892230355,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -1.2904545068740845,
      "logits/rejected": -1.4261707067489624,
      "logps/chosen": -361.19525146484375,
      "logps/rejected": -322.1033630371094,
      "loss": 0.7204,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.05613602325320244,
      "rewards/margins": 0.04490422457456589,
      "rewards/rejected": 0.011231804266571999,
      "step": 20
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 34.76216235748818,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -1.326378583908081,
      "logits/rejected": -1.334900140762329,
      "logps/chosen": -390.536376953125,
      "logps/rejected": -312.28826904296875,
      "loss": 0.7412,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.034933071583509445,
      "rewards/margins": -0.05697251483798027,
      "rewards/rejected": 0.022039445117115974,
      "step": 30
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 34.40909279988205,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -1.3933440446853638,
      "logits/rejected": -1.4404706954956055,
      "logps/chosen": -329.0964050292969,
      "logps/rejected": -291.3802490234375,
      "loss": 0.725,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.02061540260910988,
      "rewards/margins": 0.004305871669203043,
      "rewards/rejected": 0.016309529542922974,
      "step": 40
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 33.558772768765095,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": -1.3897579908370972,
      "logits/rejected": -1.4129403829574585,
      "logps/chosen": -353.4942321777344,
      "logps/rejected": -314.30755615234375,
      "loss": 0.7195,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.021332601085305214,
      "rewards/margins": 0.062744140625,
      "rewards/rejected": -0.04141153767704964,
      "step": 50
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 37.25091651222252,
      "learning_rate": 6.249999999999999e-07,
      "logits/chosen": -1.4710047245025635,
      "logits/rejected": -1.4629329442977905,
      "logps/chosen": -328.04437255859375,
      "logps/rejected": -303.51922607421875,
      "loss": 0.7164,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.010426114313304424,
      "rewards/margins": 0.02291293814778328,
      "rewards/rejected": -0.01248682476580143,
      "step": 60
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 36.76489316948809,
      "learning_rate": 7.291666666666666e-07,
      "logits/chosen": -1.3989927768707275,
      "logits/rejected": -1.3778417110443115,
      "logps/chosen": -402.1053466796875,
      "logps/rejected": -349.107421875,
      "loss": 0.7271,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.033832818269729614,
      "rewards/margins": 0.09618775546550751,
      "rewards/rejected": -0.062354933470487595,
      "step": 70
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 36.668613144227514,
      "learning_rate": 8.333333333333333e-07,
      "logits/chosen": -1.14809250831604,
      "logits/rejected": -1.3004900217056274,
      "logps/chosen": -393.3706970214844,
      "logps/rejected": -302.55267333984375,
      "loss": 0.7166,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022442013025283813,
      "rewards/margins": 0.0886504203081131,
      "rewards/rejected": -0.06620840728282928,
      "step": 80
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 37.50656065265379,
      "learning_rate": 9.374999999999999e-07,
      "logits/chosen": -1.4117727279663086,
      "logits/rejected": -1.4133552312850952,
      "logps/chosen": -344.2930603027344,
      "logps/rejected": -297.4874572753906,
      "loss": 0.7401,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.002166506601497531,
      "rewards/margins": -0.026871949434280396,
      "rewards/rejected": 0.024705441668629646,
      "step": 90
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 40.39537923810557,
      "learning_rate": 9.999463737538052e-07,
      "logits/chosen": -1.4038206338882446,
      "logits/rejected": -1.4591708183288574,
      "logps/chosen": -382.0655822753906,
      "logps/rejected": -325.4857482910156,
      "loss": 0.7098,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.009398235008120537,
      "rewards/margins": 0.012028699740767479,
      "rewards/rejected": -0.002630466129630804,
      "step": 100
    },
    {
      "epoch": 0.2093692750588851,
      "eval_logits/chosen": -1.638710618019104,
      "eval_logits/rejected": -1.6983067989349365,
      "eval_logps/chosen": -375.59967041015625,
      "eval_logps/rejected": -323.07269287109375,
      "eval_loss": 0.7161566615104675,
      "eval_rewards/accuracies": 0.5277777910232544,
      "eval_rewards/chosen": -0.010860972106456757,
      "eval_rewards/margins": 0.05658983066678047,
      "eval_rewards/rejected": -0.06745080649852753,
      "eval_runtime": 23.8466,
      "eval_samples_per_second": 83.869,
      "eval_steps_per_second": 2.642,
      "step": 100
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 31.68914195854589,
      "learning_rate": 9.993432105822034e-07,
      "logits/chosen": -1.3994014263153076,
      "logits/rejected": -1.5090693235397339,
      "logps/chosen": -380.85809326171875,
      "logps/rejected": -315.415771484375,
      "loss": 0.7244,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.0006190292770043015,
      "rewards/margins": 0.017153877764940262,
      "rewards/rejected": -0.016534846276044846,
      "step": 110
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 32.5897566012542,
      "learning_rate": 9.980706626858607e-07,
      "logits/chosen": -1.3891736268997192,
      "logits/rejected": -1.4548366069793701,
      "logps/chosen": -330.4214782714844,
      "logps/rejected": -290.9881286621094,
      "loss": 0.7015,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.002393510192632675,
      "rewards/margins": 0.015673214569687843,
      "rewards/rejected": -0.013279703445732594,
      "step": 120
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 37.434859226068475,
      "learning_rate": 9.961304359538434e-07,
      "logits/chosen": -1.3691463470458984,
      "logits/rejected": -1.4777592420578003,
      "logps/chosen": -375.37957763671875,
      "logps/rejected": -303.9295959472656,
      "loss": 0.6959,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.023477336391806602,
      "rewards/margins": 0.07401027530431747,
      "rewards/rejected": -0.05053293704986572,
      "step": 130
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 32.710975845053845,
      "learning_rate": 9.935251313189563e-07,
      "logits/chosen": -1.2984201908111572,
      "logits/rejected": -1.3567609786987305,
      "logps/chosen": -403.44390869140625,
      "logps/rejected": -315.7667541503906,
      "loss": 0.694,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.1049453616142273,
      "rewards/margins": 0.1717820167541504,
      "rewards/rejected": -0.0668366402387619,
      "step": 140
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 33.59099498042739,
      "learning_rate": 9.902582412711118e-07,
      "logits/chosen": -1.3101998567581177,
      "logits/rejected": -1.4272937774658203,
      "logps/chosen": -362.3497619628906,
      "logps/rejected": -304.9016418457031,
      "loss": 0.6869,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.030804574489593506,
      "rewards/margins": 0.16174563765525818,
      "rewards/rejected": -0.13094103336334229,
      "step": 150
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 33.64536927205325,
      "learning_rate": 9.86334145175542e-07,
      "logits/chosen": -1.372686743736267,
      "logits/rejected": -1.3871064186096191,
      "logps/chosen": -333.40252685546875,
      "logps/rejected": -309.45538330078125,
      "loss": 0.698,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02940761111676693,
      "rewards/margins": 0.14784011244773865,
      "rewards/rejected": -0.11843249946832657,
      "step": 160
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 32.519654688788634,
      "learning_rate": 9.817581034021272e-07,
      "logits/chosen": -1.469236135482788,
      "logits/rejected": -1.5464330911636353,
      "logps/chosen": -325.42401123046875,
      "logps/rejected": -279.6885986328125,
      "loss": 0.6943,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.008727139793336391,
      "rewards/margins": 0.06822283565998077,
      "rewards/rejected": -0.05949569493532181,
      "step": 170
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 31.217980241389647,
      "learning_rate": 9.765362502737097e-07,
      "logits/chosen": -1.392663598060608,
      "logits/rejected": -1.3267390727996826,
      "logps/chosen": -328.3432312011719,
      "logps/rejected": -328.40679931640625,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0019484326476231217,
      "rewards/margins": 0.09815844148397446,
      "rewards/rejected": -0.10010688006877899,
      "step": 180
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 35.19062151474422,
      "learning_rate": 9.706755858428485e-07,
      "logits/chosen": -1.317312479019165,
      "logits/rejected": -1.3148291110992432,
      "logps/chosen": -339.4427490234375,
      "logps/rejected": -339.53936767578125,
      "loss": 0.6886,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.017630616202950478,
      "rewards/margins": 0.09675482660531998,
      "rewards/rejected": -0.1143854409456253,
      "step": 190
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 38.736703713611334,
      "learning_rate": 9.641839665080363e-07,
      "logits/chosen": -1.3673009872436523,
      "logits/rejected": -1.3681590557098389,
      "logps/chosen": -341.8690490722656,
      "logps/rejected": -310.6811828613281,
      "loss": 0.6825,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.02894829586148262,
      "rewards/margins": 0.07689115405082703,
      "rewards/rejected": -0.10583944618701935,
      "step": 200
    },
    {
      "epoch": 0.4187385501177702,
      "eval_logits/chosen": -1.635803461074829,
      "eval_logits/rejected": -1.69376540184021,
      "eval_logps/chosen": -375.5799865722656,
      "eval_logps/rejected": -323.3138732910156,
      "eval_loss": 0.6841831207275391,
      "eval_rewards/accuracies": 0.579365074634552,
      "eval_rewards/chosen": -0.001013976288959384,
      "eval_rewards/margins": 0.18699844181537628,
      "eval_rewards/rejected": -0.18801240622997284,
      "eval_runtime": 22.6938,
      "eval_samples_per_second": 88.13,
      "eval_steps_per_second": 2.776,
      "step": 200
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 29.329351201852436,
      "learning_rate": 9.570700944819582e-07,
      "logits/chosen": -1.1939313411712646,
      "logits/rejected": -1.3856732845306396,
      "logps/chosen": -384.10784912109375,
      "logps/rejected": -317.24755859375,
      "loss": 0.684,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0037221908569335938,
      "rewards/margins": 0.16761450469493866,
      "rewards/rejected": -0.16389232873916626,
      "step": 210
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 29.84515748563858,
      "learning_rate": 9.493435061259129e-07,
      "logits/chosen": -1.3803870677947998,
      "logits/rejected": -1.3274301290512085,
      "logps/chosen": -313.02777099609375,
      "logps/rejected": -297.11004638671875,
      "loss": 0.6846,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.023346154019236565,
      "rewards/margins": 0.06815574318170547,
      "rewards/rejected": -0.09150189161300659,
      "step": 220
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 31.662572343473325,
      "learning_rate": 9.4101455916603e-07,
      "logits/chosen": -1.3570753335952759,
      "logits/rejected": -1.3131479024887085,
      "logps/chosen": -331.6525573730469,
      "logps/rejected": -329.0929260253906,
      "loss": 0.6834,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.025835394859313965,
      "rewards/margins": 0.16715176403522491,
      "rewards/rejected": -0.1929871290922165,
      "step": 230
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 34.26947764899909,
      "learning_rate": 9.320944188084241e-07,
      "logits/chosen": -1.3401970863342285,
      "logits/rejected": -1.3203916549682617,
      "logps/chosen": -341.75750732421875,
      "logps/rejected": -315.9219970703125,
      "loss": 0.6817,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.01570424810051918,
      "rewards/margins": 0.13407179713249207,
      "rewards/rejected": -0.11836756765842438,
      "step": 240
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 32.570026830574825,
      "learning_rate": 9.225950427718974e-07,
      "logits/chosen": -1.3138176202774048,
      "logits/rejected": -1.4214763641357422,
      "logps/chosen": -388.63800048828125,
      "logps/rejected": -327.308349609375,
      "loss": 0.662,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.034160248935222626,
      "rewards/margins": 0.23999419808387756,
      "rewards/rejected": -0.20583395659923553,
      "step": 250
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 31.875700958447943,
      "learning_rate": 9.125291652582547e-07,
      "logits/chosen": -1.171736717224121,
      "logits/rejected": -1.2927379608154297,
      "logps/chosen": -367.10638427734375,
      "logps/rejected": -301.8586730957031,
      "loss": 0.6732,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.03132578730583191,
      "rewards/margins": 0.2260134220123291,
      "rewards/rejected": -0.2573392391204834,
      "step": 260
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 29.95901746509434,
      "learning_rate": 9.019102798817195e-07,
      "logits/chosen": -1.4621995687484741,
      "logits/rejected": -1.563369631767273,
      "logps/chosen": -379.08746337890625,
      "logps/rejected": -326.1041259765625,
      "loss": 0.6799,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010546261444687843,
      "rewards/margins": 0.17825904488563538,
      "rewards/rejected": -0.16771277785301208,
      "step": 270
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 35.30660280902673,
      "learning_rate": 8.90752621580335e-07,
      "logits/chosen": -1.4563312530517578,
      "logits/rejected": -1.5124019384384155,
      "logps/chosen": -360.3949279785156,
      "logps/rejected": -296.45318603515625,
      "loss": 0.6904,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.05117585510015488,
      "rewards/margins": 0.1499311625957489,
      "rewards/rejected": -0.20110702514648438,
      "step": 280
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 30.02627471974457,
      "learning_rate": 8.79071147533597e-07,
      "logits/chosen": -1.3743504285812378,
      "logits/rejected": -1.3512884378433228,
      "logps/chosen": -341.1376647949219,
      "logps/rejected": -331.7826843261719,
      "loss": 0.6581,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.004191230051219463,
      "rewards/margins": 0.26077696681022644,
      "rewards/rejected": -0.26496821641921997,
      "step": 290
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 32.675301753581984,
      "learning_rate": 8.668815171119019e-07,
      "logits/chosen": -1.2676564455032349,
      "logits/rejected": -1.319390058517456,
      "logps/chosen": -378.872314453125,
      "logps/rejected": -322.4703369140625,
      "loss": 0.663,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.022002484649419785,
      "rewards/margins": 0.1741904467344284,
      "rewards/rejected": -0.1961929351091385,
      "step": 300
    },
    {
      "epoch": 0.6281078251766553,
      "eval_logits/chosen": -1.6317138671875,
      "eval_logits/rejected": -1.6892949342727661,
      "eval_logps/chosen": -375.532958984375,
      "eval_logps/rejected": -323.4156494140625,
      "eval_loss": 0.6616633534431458,
      "eval_rewards/accuracies": 0.60317462682724,
      "eval_rewards/chosen": 0.022496730089187622,
      "eval_rewards/margins": 0.2614184617996216,
      "eval_rewards/rejected": -0.23892174661159515,
      "eval_runtime": 22.3915,
      "eval_samples_per_second": 89.32,
      "eval_steps_per_second": 2.814,
      "step": 300
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 32.37940998012059,
      "learning_rate": 8.54200070884685e-07,
      "logits/chosen": -1.3149330615997314,
      "logits/rejected": -1.4230272769927979,
      "logps/chosen": -389.00384521484375,
      "logps/rejected": -293.93487548828125,
      "loss": 0.6471,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.028895879164338112,
      "rewards/margins": 0.29164841771125793,
      "rewards/rejected": -0.3205442726612091,
      "step": 310
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 34.74994897373754,
      "learning_rate": 8.410438087153911e-07,
      "logits/chosen": -1.2203506231307983,
      "logits/rejected": -1.28737211227417,
      "logps/chosen": -353.5164489746094,
      "logps/rejected": -297.2350769042969,
      "loss": 0.6689,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.03598624840378761,
      "rewards/margins": 0.1709606647491455,
      "rewards/rejected": -0.20694692432880402,
      "step": 320
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 27.89021442930517,
      "learning_rate": 8.274303669726426e-07,
      "logits/chosen": -1.2289667129516602,
      "logits/rejected": -1.3027209043502808,
      "logps/chosen": -351.02435302734375,
      "logps/rejected": -311.8251953125,
      "loss": 0.6675,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.005795622244477272,
      "rewards/margins": 0.3018319308757782,
      "rewards/rejected": -0.30762752890586853,
      "step": 330
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 32.72596936653152,
      "learning_rate": 8.133779948881513e-07,
      "logits/chosen": -1.3401914834976196,
      "logits/rejected": -1.387083888053894,
      "logps/chosen": -355.665771484375,
      "logps/rejected": -318.4053955078125,
      "loss": 0.6557,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04015441983938217,
      "rewards/margins": 0.22958505153656006,
      "rewards/rejected": -0.2697394788265228,
      "step": 340
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 35.42610267799739,
      "learning_rate": 7.989055300930704e-07,
      "logits/chosen": -1.3034248352050781,
      "logits/rejected": -1.3343088626861572,
      "logps/chosen": -340.1492614746094,
      "logps/rejected": -289.6612854003906,
      "loss": 0.6554,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1225026398897171,
      "rewards/margins": 0.12164878845214844,
      "rewards/rejected": -0.24415142834186554,
      "step": 350
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 29.02918447808193,
      "learning_rate": 7.840323733655778e-07,
      "logits/chosen": -1.2503941059112549,
      "logits/rejected": -1.4153048992156982,
      "logps/chosen": -406.9992980957031,
      "logps/rejected": -322.87994384765625,
      "loss": 0.6565,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.024089232087135315,
      "rewards/margins": 0.31749090552330017,
      "rewards/rejected": -0.29340168833732605,
      "step": 360
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 33.710516075595095,
      "learning_rate": 7.687784626235447e-07,
      "logits/chosen": -1.312897801399231,
      "logits/rejected": -1.3840960264205933,
      "logps/chosen": -369.8164978027344,
      "logps/rejected": -291.6910705566406,
      "loss": 0.643,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.02891460619866848,
      "rewards/margins": 0.3528975546360016,
      "rewards/rejected": -0.3239828944206238,
      "step": 370
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 30.30908317538956,
      "learning_rate": 7.531642461971514e-07,
      "logits/chosen": -1.4397472143173218,
      "logits/rejected": -1.5266616344451904,
      "logps/chosen": -368.24224853515625,
      "logps/rejected": -316.00567626953125,
      "loss": 0.6553,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07685438543558121,
      "rewards/margins": 0.29695773124694824,
      "rewards/rejected": -0.37381213903427124,
      "step": 380
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 34.60440393056815,
      "learning_rate": 7.372106554172801e-07,
      "logits/chosen": -1.3900893926620483,
      "logits/rejected": -1.348008394241333,
      "logps/chosen": -369.2901306152344,
      "logps/rejected": -338.5638427734375,
      "loss": 0.661,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0064844414591789246,
      "rewards/margins": 0.22314131259918213,
      "rewards/rejected": -0.22962574660778046,
      "step": 390
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 31.77115164859215,
      "learning_rate": 7.209390765564318e-07,
      "logits/chosen": -1.4996167421340942,
      "logits/rejected": -1.4596645832061768,
      "logps/chosen": -308.35113525390625,
      "logps/rejected": -281.60693359375,
      "loss": 0.6547,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04348212480545044,
      "rewards/margins": 0.27009111642837524,
      "rewards/rejected": -0.3135732412338257,
      "step": 400
    },
    {
      "epoch": 0.8374771002355405,
      "eval_logits/chosen": -1.6414421796798706,
      "eval_logits/rejected": -1.6979553699493408,
      "eval_logps/chosen": -375.57781982421875,
      "eval_logps/rejected": -323.6410217285156,
      "eval_loss": 0.6590762734413147,
      "eval_rewards/accuracies": 0.6388888955116272,
      "eval_rewards/chosen": 7.240355625981465e-05,
      "eval_rewards/margins": 0.3516833484172821,
      "eval_rewards/rejected": -0.3516109585762024,
      "eval_runtime": 22.0937,
      "eval_samples_per_second": 90.524,
      "eval_steps_per_second": 2.851,
      "step": 400
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 31.789442088224007,
      "learning_rate": 7.043713221597773e-07,
      "logits/chosen": -1.2454522848129272,
      "logits/rejected": -1.4237592220306396,
      "logps/chosen": -399.0449523925781,
      "logps/rejected": -326.0782775878906,
      "loss": 0.6462,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.014972883276641369,
      "rewards/margins": 0.2356092631816864,
      "rewards/rejected": -0.22063639760017395,
      "step": 410
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 32.68488619173213,
      "learning_rate": 6.875296018047809e-07,
      "logits/chosen": -1.5021634101867676,
      "logits/rejected": -1.4257535934448242,
      "logps/chosen": -349.61553955078125,
      "logps/rejected": -340.30975341796875,
      "loss": 0.6371,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10846195369958878,
      "rewards/margins": 0.26752999424934387,
      "rewards/rejected": -0.37599191069602966,
      "step": 420
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 29.283057753738706,
      "learning_rate": 6.704364923285857e-07,
      "logits/chosen": -1.3049055337905884,
      "logits/rejected": -1.4615538120269775,
      "logps/chosen": -387.3922119140625,
      "logps/rejected": -299.621337890625,
      "loss": 0.6539,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06501199305057526,
      "rewards/margins": 0.30248281359672546,
      "rewards/rejected": -0.3674947917461395,
      "step": 430
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 30.29029590057011,
      "learning_rate": 6.531149075630796e-07,
      "logits/chosen": -1.2950228452682495,
      "logits/rejected": -1.3765883445739746,
      "logps/chosen": -359.27691650390625,
      "logps/rejected": -293.13104248046875,
      "loss": 0.6508,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.05934730917215347,
      "rewards/margins": 0.2306128442287445,
      "rewards/rejected": -0.28996017575263977,
      "step": 440
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 31.040736224709473,
      "learning_rate": 6.355880676182085e-07,
      "logits/chosen": -1.4344260692596436,
      "logits/rejected": -1.3737527132034302,
      "logps/chosen": -357.5885314941406,
      "logps/rejected": -332.05499267578125,
      "loss": 0.6653,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04119770601391792,
      "rewards/margins": 0.2782004475593567,
      "rewards/rejected": -0.31939810514450073,
      "step": 450
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 31.5228211898507,
      "learning_rate": 6.178794677547137e-07,
      "logits/chosen": -1.2749378681182861,
      "logits/rejected": -1.4267973899841309,
      "logps/chosen": -346.6052551269531,
      "logps/rejected": -290.1454162597656,
      "loss": 0.6415,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.026611875742673874,
      "rewards/margins": 0.3765586316585541,
      "rewards/rejected": -0.3499467074871063,
      "step": 460
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 31.660312290338712,
      "learning_rate": 6.000128468880222e-07,
      "logits/chosen": -1.3701679706573486,
      "logits/rejected": -1.390453815460205,
      "logps/chosen": -369.3509826660156,
      "logps/rejected": -323.2626037597656,
      "loss": 0.6358,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04081202298402786,
      "rewards/margins": 0.33853498101234436,
      "rewards/rejected": -0.37934696674346924,
      "step": 470
    },
    {
      "epoch": 1.0049725202826485,
      "grad_norm": 27.48266116042723,
      "learning_rate": 5.820121557655108e-07,
      "logits/chosen": -1.2848116159439087,
      "logits/rejected": -1.3119057416915894,
      "logps/chosen": -359.86102294921875,
      "logps/rejected": -313.0788269042969,
      "loss": 0.6459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.048723649233579636,
      "rewards/margins": 0.27731940150260925,
      "rewards/rejected": -0.3260430693626404,
      "step": 480
    },
    {
      "epoch": 1.025909447788537,
      "grad_norm": 31.66810532106133,
      "learning_rate": 5.639015248598023e-07,
      "logits/chosen": -1.34721839427948,
      "logits/rejected": -1.449410080909729,
      "logps/chosen": -361.23046875,
      "logps/rejected": -296.3492126464844,
      "loss": 0.6519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09297003597021103,
      "rewards/margins": 0.209228515625,
      "rewards/rejected": -0.302198588848114,
      "step": 490
    },
    {
      "epoch": 1.0468463752944255,
      "grad_norm": 31.533076924353868,
      "learning_rate": 5.457052320211339e-07,
      "logits/chosen": -1.3988004922866821,
      "logits/rejected": -1.4485900402069092,
      "logps/chosen": -370.951904296875,
      "logps/rejected": -314.5948181152344,
      "loss": 0.6456,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10979924350976944,
      "rewards/margins": 0.23507650196552277,
      "rewards/rejected": -0.3448757231235504,
      "step": 500
    },
    {
      "epoch": 1.0468463752944255,
      "eval_logits/chosen": -1.636523962020874,
      "eval_logits/rejected": -1.6931382417678833,
      "eval_logps/chosen": -375.5513916015625,
      "eval_logps/rejected": -323.6509704589844,
      "eval_loss": 0.6429626941680908,
      "eval_rewards/accuracies": 0.6666666865348816,
      "eval_rewards/chosen": 0.01327058020979166,
      "eval_rewards/margins": 0.36985495686531067,
      "eval_rewards/rejected": -0.356584370136261,
      "eval_runtime": 22.4458,
      "eval_samples_per_second": 89.104,
      "eval_steps_per_second": 2.807,
      "step": 500
    },
    {
      "epoch": 1.067783302800314,
      "grad_norm": 30.307455199305462,
      "learning_rate": 5.274476699321637e-07,
      "logits/chosen": -1.4101717472076416,
      "logits/rejected": -1.420190453529358,
      "logps/chosen": -327.46893310546875,
      "logps/rejected": -302.391357421875,
      "loss": 0.6345,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04776526987552643,
      "rewards/margins": 0.2989855408668518,
      "rewards/rejected": -0.3467508554458618,
      "step": 510
    },
    {
      "epoch": 1.0887202303062025,
      "grad_norm": 29.722389429702496,
      "learning_rate": 5.091533134088387e-07,
      "logits/chosen": -1.4273402690887451,
      "logits/rejected": -1.4231868982315063,
      "logps/chosen": -323.42376708984375,
      "logps/rejected": -304.4673767089844,
      "loss": 0.6284,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0829726904630661,
      "rewards/margins": 0.24710920453071594,
      "rewards/rejected": -0.33008190989494324,
      "step": 520
    },
    {
      "epoch": 1.109657157812091,
      "grad_norm": 28.885339598508185,
      "learning_rate": 4.908466865911614e-07,
      "logits/chosen": -1.4464526176452637,
      "logits/rejected": -1.5408886671066284,
      "logps/chosen": -344.06689453125,
      "logps/rejected": -292.1816101074219,
      "loss": 0.6328,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03225661441683769,
      "rewards/margins": 0.38520506024360657,
      "rewards/rejected": -0.41746169328689575,
      "step": 530
    },
    {
      "epoch": 1.1305940853179797,
      "grad_norm": 29.441675570164428,
      "learning_rate": 4.7255233006783624e-07,
      "logits/chosen": -1.3947404623031616,
      "logits/rejected": -1.328045129776001,
      "logps/chosen": -318.3937072753906,
      "logps/rejected": -280.65435791015625,
      "loss": 0.6372,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03985299542546272,
      "rewards/margins": 0.33030450344085693,
      "rewards/rejected": -0.37015751004219055,
      "step": 540
    },
    {
      "epoch": 1.151531012823868,
      "grad_norm": 29.84505193266847,
      "learning_rate": 4.5429476797886617e-07,
      "logits/chosen": -1.3157615661621094,
      "logits/rejected": -1.463357925415039,
      "logps/chosen": -367.59564208984375,
      "logps/rejected": -284.3589172363281,
      "loss": 0.6256,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04548700898885727,
      "rewards/margins": 0.312529981136322,
      "rewards/rejected": -0.3580169677734375,
      "step": 550
    },
    {
      "epoch": 1.1724679403297567,
      "grad_norm": 33.58402259379722,
      "learning_rate": 4.3609847514019763e-07,
      "logits/chosen": -1.3491549491882324,
      "logits/rejected": -1.3917362689971924,
      "logps/chosen": -355.3868408203125,
      "logps/rejected": -311.2879943847656,
      "loss": 0.6294,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.005126100964844227,
      "rewards/margins": 0.33152928948402405,
      "rewards/rejected": -0.3264032006263733,
      "step": 560
    },
    {
      "epoch": 1.193404867835645,
      "grad_norm": 29.80795991678476,
      "learning_rate": 4.179878442344892e-07,
      "logits/chosen": -1.4194284677505493,
      "logits/rejected": -1.371366262435913,
      "logps/chosen": -326.87066650390625,
      "logps/rejected": -321.23785400390625,
      "loss": 0.6441,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.0237637497484684,
      "rewards/margins": 0.33783239126205444,
      "rewards/rejected": -0.31406864523887634,
      "step": 570
    },
    {
      "epoch": 1.2143417953415336,
      "grad_norm": 30.101844117489865,
      "learning_rate": 3.9998715311197783e-07,
      "logits/chosen": -1.452893853187561,
      "logits/rejected": -1.460272192955017,
      "logps/chosen": -346.24932861328125,
      "logps/rejected": -292.27667236328125,
      "loss": 0.6258,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.029639745131134987,
      "rewards/margins": 0.4036487638950348,
      "rewards/rejected": -0.43328848481178284,
      "step": 580
    },
    {
      "epoch": 1.235278722847422,
      "grad_norm": 28.648910640416673,
      "learning_rate": 3.821205322452863e-07,
      "logits/chosen": -1.2438939809799194,
      "logits/rejected": -1.3377447128295898,
      "logps/chosen": -380.3300476074219,
      "logps/rejected": -317.4806823730469,
      "loss": 0.6249,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0937061458826065,
      "rewards/margins": 0.34427446126937866,
      "rewards/rejected": -0.43798065185546875,
      "step": 590
    },
    {
      "epoch": 1.2562156503533106,
      "grad_norm": 34.234491670474995,
      "learning_rate": 3.6441193238179146e-07,
      "logits/chosen": -1.4130090475082397,
      "logits/rejected": -1.340057611465454,
      "logps/chosen": -377.28228759765625,
      "logps/rejected": -362.57879638671875,
      "loss": 0.6054,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08141253888607025,
      "rewards/margins": 0.26778295636177063,
      "rewards/rejected": -0.3491954803466797,
      "step": 600
    },
    {
      "epoch": 1.2562156503533106,
      "eval_logits/chosen": -1.6430615186691284,
      "eval_logits/rejected": -1.6991180181503296,
      "eval_logps/chosen": -375.643798828125,
      "eval_logps/rejected": -323.71673583984375,
      "eval_loss": 0.6423465609550476,
      "eval_rewards/accuracies": 0.6349206566810608,
      "eval_rewards/chosen": -0.032912444323301315,
      "eval_rewards/margins": 0.3565501272678375,
      "eval_rewards/rejected": -0.38946259021759033,
      "eval_runtime": 22.0916,
      "eval_samples_per_second": 90.532,
      "eval_steps_per_second": 2.852,
      "step": 600
    },
    {
      "epoch": 1.2771525778591992,
      "grad_norm": 31.96916362076228,
      "learning_rate": 3.4688509243692034e-07,
      "logits/chosen": -1.3404086828231812,
      "logits/rejected": -1.401881217956543,
      "logps/chosen": -348.0643005371094,
      "logps/rejected": -274.1236267089844,
      "loss": 0.6089,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.05336350202560425,
      "rewards/margins": 0.5003852248191833,
      "rewards/rejected": -0.4470217823982239,
      "step": 610
    },
    {
      "epoch": 1.2980895053650876,
      "grad_norm": 27.687942045456108,
      "learning_rate": 3.295635076714144e-07,
      "logits/chosen": -1.1929457187652588,
      "logits/rejected": -1.3312275409698486,
      "logps/chosen": -339.0008544921875,
      "logps/rejected": -282.86273193359375,
      "loss": 0.6173,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04629793018102646,
      "rewards/margins": 0.3611483573913574,
      "rewards/rejected": -0.4074462950229645,
      "step": 620
    },
    {
      "epoch": 1.3190264328709762,
      "grad_norm": 30.364814060074504,
      "learning_rate": 3.12470398195219e-07,
      "logits/chosen": -1.4581263065338135,
      "logits/rejected": -1.4359983205795288,
      "logps/chosen": -352.0130310058594,
      "logps/rejected": -319.5924987792969,
      "loss": 0.6124,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.0860348716378212,
      "rewards/margins": 0.4933602213859558,
      "rewards/rejected": -0.4073253571987152,
      "step": 630
    },
    {
      "epoch": 1.3399633603768648,
      "grad_norm": 28.207562684445328,
      "learning_rate": 2.956286778402226e-07,
      "logits/chosen": -1.3643245697021484,
      "logits/rejected": -1.3367928266525269,
      "logps/chosen": -329.8095397949219,
      "logps/rejected": -325.6297912597656,
      "loss": 0.6194,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03843813017010689,
      "rewards/margins": 0.4010621905326843,
      "rewards/rejected": -0.4395003318786621,
      "step": 640
    },
    {
      "epoch": 1.3609002878827532,
      "grad_norm": 31.931835373695627,
      "learning_rate": 2.7906092344356826e-07,
      "logits/chosen": -1.4707378149032593,
      "logits/rejected": -1.4725669622421265,
      "logps/chosen": -321.7577819824219,
      "logps/rejected": -296.30072021484375,
      "loss": 0.6295,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.20691867172718048,
      "rewards/margins": 0.24088874459266663,
      "rewards/rejected": -0.4478074014186859,
      "step": 650
    },
    {
      "epoch": 1.3818372153886418,
      "grad_norm": 33.26417313890622,
      "learning_rate": 2.6278934458271996e-07,
      "logits/chosen": -1.280631422996521,
      "logits/rejected": -1.3918514251708984,
      "logps/chosen": -319.20391845703125,
      "logps/rejected": -294.9554138183594,
      "loss": 0.6281,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.15276700258255005,
      "rewards/margins": 0.14049580693244934,
      "rewards/rejected": -0.293262779712677,
      "step": 660
    },
    {
      "epoch": 1.4027741428945302,
      "grad_norm": 27.692247842703622,
      "learning_rate": 2.468357538028487e-07,
      "logits/chosen": -1.3904974460601807,
      "logits/rejected": -1.4726046323776245,
      "logps/chosen": -351.35614013671875,
      "logps/rejected": -299.1191101074219,
      "loss": 0.6141,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.012430467642843723,
      "rewards/margins": 0.4077607989311218,
      "rewards/rejected": -0.39533036947250366,
      "step": 670
    },
    {
      "epoch": 1.4237110704004188,
      "grad_norm": 32.32896888329579,
      "learning_rate": 2.312215373764551e-07,
      "logits/chosen": -1.3210734128952026,
      "logits/rejected": -1.2972867488861084,
      "logps/chosen": -357.87725830078125,
      "logps/rejected": -346.83172607421875,
      "loss": 0.612,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.030674666166305542,
      "rewards/margins": 0.2985823154449463,
      "rewards/rejected": -0.32925695180892944,
      "step": 680
    },
    {
      "epoch": 1.4446479979063072,
      "grad_norm": 26.559915261204598,
      "learning_rate": 2.1596762663442213e-07,
      "logits/chosen": -1.353945016860962,
      "logits/rejected": -1.331405758857727,
      "logps/chosen": -359.0257263183594,
      "logps/rejected": -302.020751953125,
      "loss": 0.6143,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.09165488183498383,
      "rewards/margins": 0.32221120595932007,
      "rewards/rejected": -0.4138661324977875,
      "step": 690
    },
    {
      "epoch": 1.4655849254121958,
      "grad_norm": 28.923951135790823,
      "learning_rate": 2.0109446990692963e-07,
      "logits/chosen": -1.3177390098571777,
      "logits/rejected": -1.2676212787628174,
      "logps/chosen": -382.9945373535156,
      "logps/rejected": -382.7451171875,
      "loss": 0.6129,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.00983230210840702,
      "rewards/margins": 0.44890522956848145,
      "rewards/rejected": -0.43907293677330017,
      "step": 700
    },
    {
      "epoch": 1.4655849254121958,
      "eval_logits/chosen": -1.6413902044296265,
      "eval_logits/rejected": -1.6979249715805054,
      "eval_logps/chosen": -375.667724609375,
      "eval_logps/rejected": -323.7745361328125,
      "eval_loss": 0.6431064009666443,
      "eval_rewards/accuracies": 0.6349206566810608,
      "eval_rewards/chosen": -0.044887345284223557,
      "eval_rewards/margins": 0.373458594083786,
      "eval_rewards/rejected": -0.41834595799446106,
      "eval_runtime": 22.4693,
      "eval_samples_per_second": 89.01,
      "eval_steps_per_second": 2.804,
      "step": 700
    },
    {
      "epoch": 1.4865218529180844,
      "grad_norm": 29.793015750302608,
      "learning_rate": 1.8662200511184872e-07,
      "logits/chosen": -1.5391782522201538,
      "logits/rejected": -1.5692270994186401,
      "logps/chosen": -353.60504150390625,
      "logps/rejected": -327.60333251953125,
      "loss": 0.6413,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08452965319156647,
      "rewards/margins": 0.3078867793083191,
      "rewards/rejected": -0.39241647720336914,
      "step": 710
    },
    {
      "epoch": 1.5074587804239727,
      "grad_norm": 30.154215075312383,
      "learning_rate": 1.725696330273575e-07,
      "logits/chosen": -1.4344627857208252,
      "logits/rejected": -1.4114643335342407,
      "logps/chosen": -367.28668212890625,
      "logps/rejected": -330.210205078125,
      "loss": 0.6152,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0991826206445694,
      "rewards/margins": 0.3690223693847656,
      "rewards/rejected": -0.46820497512817383,
      "step": 720
    },
    {
      "epoch": 1.5283957079298613,
      "grad_norm": 32.49028184373073,
      "learning_rate": 1.589561912846089e-07,
      "logits/chosen": -1.3968040943145752,
      "logits/rejected": -1.4468437433242798,
      "logps/chosen": -339.4754943847656,
      "logps/rejected": -298.7085266113281,
      "loss": 0.6139,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04794057086110115,
      "rewards/margins": 0.3862064480781555,
      "rewards/rejected": -0.43414703011512756,
      "step": 730
    },
    {
      "epoch": 1.54933263543575,
      "grad_norm": 29.612038431579933,
      "learning_rate": 1.4579992911531496e-07,
      "logits/chosen": -1.2432403564453125,
      "logits/rejected": -1.2390873432159424,
      "logps/chosen": -379.5582275390625,
      "logps/rejected": -332.1794128417969,
      "loss": 0.6205,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.019659820944070816,
      "rewards/margins": 0.3802977204322815,
      "rewards/rejected": -0.360637903213501,
      "step": 740
    },
    {
      "epoch": 1.5702695629416383,
      "grad_norm": 28.119711308052274,
      "learning_rate": 1.3311848288809813e-07,
      "logits/chosen": -1.4195433855056763,
      "logits/rejected": -1.490561842918396,
      "logps/chosen": -359.9119873046875,
      "logps/rejected": -322.4441833496094,
      "loss": 0.6118,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0712754875421524,
      "rewards/margins": 0.2602451741695404,
      "rewards/rejected": -0.3315206468105316,
      "step": 750
    },
    {
      "epoch": 1.5912064904475267,
      "grad_norm": 29.268690125436255,
      "learning_rate": 1.209288524664029e-07,
      "logits/chosen": -1.229116439819336,
      "logits/rejected": -1.2306039333343506,
      "logps/chosen": -435.7142028808594,
      "logps/rejected": -397.52044677734375,
      "loss": 0.5971,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.11046306788921356,
      "rewards/margins": 0.4868060052394867,
      "rewards/rejected": -0.3763429522514343,
      "step": 760
    },
    {
      "epoch": 1.6121434179534153,
      "grad_norm": 28.29047720088543,
      "learning_rate": 1.0924737841966497e-07,
      "logits/chosen": -1.220485806465149,
      "logits/rejected": -1.4240710735321045,
      "logps/chosen": -396.7095947265625,
      "logps/rejected": -319.75958251953125,
      "loss": 0.6247,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.055124759674072266,
      "rewards/margins": 0.39260005950927734,
      "rewards/rejected": -0.44772475957870483,
      "step": 770
    },
    {
      "epoch": 1.633080345459304,
      "grad_norm": 29.17224056970089,
      "learning_rate": 9.808972011828054e-08,
      "logits/chosen": -1.2811758518218994,
      "logits/rejected": -1.3020236492156982,
      "logps/chosen": -384.5751647949219,
      "logps/rejected": -329.9205017089844,
      "loss": 0.626,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.14039060473442078,
      "rewards/margins": 0.46962863206863403,
      "rewards/rejected": -0.32923805713653564,
      "step": 780
    },
    {
      "epoch": 1.6540172729651923,
      "grad_norm": 33.75145778470617,
      "learning_rate": 8.747083474174527e-08,
      "logits/chosen": -1.3885407447814941,
      "logits/rejected": -1.4309613704681396,
      "logps/chosen": -365.4181823730469,
      "logps/rejected": -322.75054931640625,
      "loss": 0.6123,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04203175753355026,
      "rewards/margins": 0.3509315550327301,
      "rewards/rejected": -0.39296332001686096,
      "step": 790
    },
    {
      "epoch": 1.674954200471081,
      "grad_norm": 25.187206733853984,
      "learning_rate": 7.740495722810269e-08,
      "logits/chosen": -1.2112213373184204,
      "logits/rejected": -1.3428361415863037,
      "logps/chosen": -420.00830078125,
      "logps/rejected": -358.3287353515625,
      "loss": 0.5972,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.01959555223584175,
      "rewards/margins": 0.5057805776596069,
      "rewards/rejected": -0.4861849844455719,
      "step": 800
    },
    {
      "epoch": 1.674954200471081,
      "eval_logits/chosen": -1.639930009841919,
      "eval_logits/rejected": -1.6965380907058716,
      "eval_logps/chosen": -375.7169494628906,
      "eval_logps/rejected": -323.76556396484375,
      "eval_loss": 0.6384284496307373,
      "eval_rewards/accuracies": 0.6428571343421936,
      "eval_rewards/chosen": -0.06950785219669342,
      "eval_rewards/margins": 0.34436169266700745,
      "eval_rewards/rejected": -0.41386955976486206,
      "eval_runtime": 21.8454,
      "eval_samples_per_second": 91.552,
      "eval_steps_per_second": 2.884,
      "step": 800
    },
    {
      "epoch": 1.6958911279769695,
      "grad_norm": 30.26944195604407,
      "learning_rate": 6.790558119157597e-08,
      "logits/chosen": -1.3412492275238037,
      "logits/rejected": -1.3922173976898193,
      "logps/chosen": -375.84429931640625,
      "logps/rejected": -326.71417236328125,
      "loss": 0.6295,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.015008327551186085,
      "rewards/margins": 0.4259034991264343,
      "rewards/rejected": -0.44091176986694336,
      "step": 810
    },
    {
      "epoch": 1.7168280554828579,
      "grad_norm": 30.885265100170944,
      "learning_rate": 5.898544083397e-08,
      "logits/chosen": -1.4171245098114014,
      "logits/rejected": -1.5250798463821411,
      "logps/chosen": -388.115478515625,
      "logps/rejected": -320.20819091796875,
      "loss": 0.6055,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.05425529554486275,
      "rewards/margins": 0.4603540003299713,
      "rewards/rejected": -0.5146092176437378,
      "step": 820
    },
    {
      "epoch": 1.7377649829887463,
      "grad_norm": 32.971600431805705,
      "learning_rate": 5.065649387408705e-08,
      "logits/chosen": -1.3083207607269287,
      "logits/rejected": -1.2884520292282104,
      "logps/chosen": -338.13909912109375,
      "logps/rejected": -327.7836608886719,
      "loss": 0.6083,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.11997182667255402,
      "rewards/margins": 0.3927108347415924,
      "rewards/rejected": -0.5126826167106628,
      "step": 830
    },
    {
      "epoch": 1.7587019104946349,
      "grad_norm": 29.20392566444622,
      "learning_rate": 4.292990551804171e-08,
      "logits/chosen": -1.5097484588623047,
      "logits/rejected": -1.4066120386123657,
      "logps/chosen": -314.53338623046875,
      "logps/rejected": -309.9684143066406,
      "loss": 0.6346,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06001155450940132,
      "rewards/margins": 0.30454209446907043,
      "rewards/rejected": -0.36455363035202026,
      "step": 840
    },
    {
      "epoch": 1.7796388380005235,
      "grad_norm": 30.280553572438347,
      "learning_rate": 3.581603349196371e-08,
      "logits/chosen": -1.3757822513580322,
      "logits/rejected": -1.3954650163650513,
      "logps/chosen": -331.03826904296875,
      "logps/rejected": -323.1974182128906,
      "loss": 0.6233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10794013738632202,
      "rewards/margins": 0.2812719941139221,
      "rewards/rejected": -0.38921213150024414,
      "step": 850
    },
    {
      "epoch": 1.8005757655064119,
      "grad_norm": 28.100566523523845,
      "learning_rate": 2.9324414157151367e-08,
      "logits/chosen": -1.3457200527191162,
      "logits/rejected": -1.4598686695098877,
      "logps/chosen": -356.97216796875,
      "logps/rejected": -286.18865966796875,
      "loss": 0.6215,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.05811784416437149,
      "rewards/margins": 0.35237258672714233,
      "rewards/rejected": -0.4104904234409332,
      "step": 860
    },
    {
      "epoch": 1.8215126930123005,
      "grad_norm": 26.46890964804442,
      "learning_rate": 2.3463749726290284e-08,
      "logits/chosen": -1.4204833507537842,
      "logits/rejected": -1.5046513080596924,
      "logps/chosen": -404.6219177246094,
      "logps/rejected": -336.10150146484375,
      "loss": 0.6261,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02932531200349331,
      "rewards/margins": 0.39560848474502563,
      "rewards/rejected": -0.366283118724823,
      "step": 870
    },
    {
      "epoch": 1.842449620518189,
      "grad_norm": 31.701066186197266,
      "learning_rate": 1.824189659787284e-08,
      "logits/chosen": -1.3159443140029907,
      "logits/rejected": -1.3131808042526245,
      "logps/chosen": -328.07098388671875,
      "logps/rejected": -308.87701416015625,
      "loss": 0.6266,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0714392215013504,
      "rewards/margins": 0.3882763683795929,
      "rewards/rejected": -0.45971551537513733,
      "step": 880
    },
    {
      "epoch": 1.8633865480240774,
      "grad_norm": 31.12356616666238,
      "learning_rate": 1.3665854824458035e-08,
      "logits/chosen": -1.3845950365066528,
      "logits/rejected": -1.434497594833374,
      "logps/chosen": -376.0118103027344,
      "logps/rejected": -331.4889831542969,
      "loss": 0.6095,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.061675846576690674,
      "rewards/margins": 0.3783223330974579,
      "rewards/rejected": -0.43999820947647095,
      "step": 890
    },
    {
      "epoch": 1.8843234755299658,
      "grad_norm": 34.05071913262595,
      "learning_rate": 9.741758728888217e-09,
      "logits/chosen": -1.2828714847564697,
      "logits/rejected": -1.3924423456192017,
      "logps/chosen": -401.4546203613281,
      "logps/rejected": -317.35443115234375,
      "loss": 0.6207,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.004245785064995289,
      "rewards/margins": 0.4051188826560974,
      "rewards/rejected": -0.40936464071273804,
      "step": 900
    },
    {
      "epoch": 1.8843234755299658,
      "eval_logits/chosen": -1.6407170295715332,
      "eval_logits/rejected": -1.6976144313812256,
      "eval_logps/chosen": -375.70330810546875,
      "eval_logps/rejected": -323.78216552734375,
      "eval_loss": 0.6362326145172119,
      "eval_rewards/accuracies": 0.6785714030265808,
      "eval_rewards/chosen": -0.06266693770885468,
      "eval_rewards/margins": 0.3595210015773773,
      "eval_rewards/rejected": -0.4221878945827484,
      "eval_runtime": 22.2411,
      "eval_samples_per_second": 89.923,
      "eval_steps_per_second": 2.833,
      "step": 900
    },
    {
      "epoch": 1.9052604030358546,
      "grad_norm": 28.324746122618627,
      "learning_rate": 6.474868681043577e-09,
      "logits/chosen": -1.232722282409668,
      "logits/rejected": -1.3370740413665771,
      "logps/chosen": -330.0971984863281,
      "logps/rejected": -274.8087463378906,
      "loss": 0.6315,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04480002075433731,
      "rewards/margins": 0.4823727011680603,
      "rewards/rejected": -0.5271727442741394,
      "step": 910
    },
    {
      "epoch": 1.926197330541743,
      "grad_norm": 32.98699943344935,
      "learning_rate": 3.869564046156459e-09,
      "logits/chosen": -1.3717395067214966,
      "logits/rejected": -1.4535510540008545,
      "logps/chosen": -373.8208923339844,
      "logps/rejected": -311.4999084472656,
      "loss": 0.6198,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06762387603521347,
      "rewards/margins": 0.35773369669914246,
      "rewards/rejected": -0.4253575801849365,
      "step": 920
    },
    {
      "epoch": 1.9471342580476314,
      "grad_norm": 27.5401666731791,
      "learning_rate": 1.929337314139412e-09,
      "logits/chosen": -1.2926667928695679,
      "logits/rejected": -1.3644298315048218,
      "logps/chosen": -362.7403869628906,
      "logps/rejected": -317.42303466796875,
      "loss": 0.6124,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01872597262263298,
      "rewards/margins": 0.4235456883907318,
      "rewards/rejected": -0.40481966733932495,
      "step": 930
    },
    {
      "epoch": 1.96807118555352,
      "grad_norm": 31.615150149296902,
      "learning_rate": 6.567894177967325e-10,
      "logits/chosen": -1.3021799325942993,
      "logits/rejected": -1.3374966382980347,
      "logps/chosen": -324.93255615234375,
      "logps/rejected": -273.21929931640625,
      "loss": 0.6255,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05069868639111519,
      "rewards/margins": 0.33984681963920593,
      "rewards/rejected": -0.390545517206192,
      "step": 940
    },
    {
      "epoch": 1.9890081130594086,
      "grad_norm": 29.09184931904493,
      "learning_rate": 5.3626246194704575e-11,
      "logits/chosen": -1.3507764339447021,
      "logits/rejected": -1.4648717641830444,
      "logps/chosen": -361.6529235839844,
      "logps/rejected": -296.1907043457031,
      "loss": 0.6169,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08508863300085068,
      "rewards/margins": 0.3246951401233673,
      "rewards/rejected": -0.40978384017944336,
      "step": 950
    },
    {
      "epoch": 1.9973828840617638,
      "step": 954,
      "total_flos": 0.0,
      "train_loss": 0.6516540072998911,
      "train_runtime": 5944.7081,
      "train_samples_per_second": 20.568,
      "train_steps_per_second": 0.16
    }
  ],
  "logging_steps": 10,
  "max_steps": 954,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}