{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 11.773019635010494,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -1.4459120035171509,
      "logits/rejected": -1.4473698139190674,
      "logps/chosen": -7.983118534088135,
      "logps/rejected": -8.156193733215332,
      "loss": 8.9797,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.983118534088135,
      "rewards/margins": 0.17307573556900024,
      "rewards/rejected": -8.156193733215332,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 9.709040224393025,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -1.4474694728851318,
      "logits/rejected": -1.438397765159607,
      "logps/chosen": -8.046891212463379,
      "logps/rejected": -7.96016788482666,
      "loss": 8.9815,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -8.046891212463379,
      "rewards/margins": -0.08672460168600082,
      "rewards/rejected": -7.96016788482666,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 16.89107530887862,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -1.4477674961090088,
      "logits/rejected": -1.4442229270935059,
      "logps/chosen": -7.852339267730713,
      "logps/rejected": -7.866924285888672,
      "loss": 8.8904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.852339267730713,
      "rewards/margins": 0.014585095457732677,
      "rewards/rejected": -7.866924285888672,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 14.261360231352997,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -1.4316000938415527,
      "logits/rejected": -1.435153841972351,
      "logps/chosen": -8.180787086486816,
      "logps/rejected": -8.204398155212402,
      "loss": 9.0475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.180787086486816,
      "rewards/margins": 0.023611638695001602,
      "rewards/rejected": -8.204398155212402,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 13.453269431524436,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -1.479134440422058,
      "logits/rejected": -1.4693964719772339,
      "logps/chosen": -8.08868408203125,
      "logps/rejected": -7.990322113037109,
      "loss": 9.1249,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -8.08868408203125,
      "rewards/margins": -0.09836126863956451,
      "rewards/rejected": -7.990322113037109,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 11.378313696016495,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -1.447307825088501,
      "logits/rejected": -1.4355913400650024,
      "logps/chosen": -7.789179801940918,
      "logps/rejected": -7.710352897644043,
      "loss": 9.0197,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -7.789179801940918,
      "rewards/margins": -0.0788281261920929,
      "rewards/rejected": -7.710352897644043,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 10.492170948688017,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -1.4480352401733398,
      "logits/rejected": -1.421820044517517,
      "logps/chosen": -8.031718254089355,
      "logps/rejected": -7.896989345550537,
      "loss": 8.9858,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -8.031718254089355,
      "rewards/margins": -0.13472941517829895,
      "rewards/rejected": -7.896989345550537,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 9.438874148034836,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -1.4374274015426636,
      "logits/rejected": -1.4498326778411865,
      "logps/chosen": -7.972153663635254,
      "logps/rejected": -8.157554626464844,
      "loss": 9.013,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -7.972153663635254,
      "rewards/margins": 0.185401052236557,
      "rewards/rejected": -8.157554626464844,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 10.665489074479519,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -1.4577468633651733,
      "logits/rejected": -1.45815110206604,
      "logps/chosen": -8.014823913574219,
      "logps/rejected": -7.8058881759643555,
      "loss": 8.9894,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -8.014823913574219,
      "rewards/margins": -0.20893602073192596,
      "rewards/rejected": -7.8058881759643555,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 14.935150052502319,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -1.4442975521087646,
      "logits/rejected": -1.461060881614685,
      "logps/chosen": -8.208638191223145,
      "logps/rejected": -8.235433578491211,
      "loss": 9.0806,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -8.208638191223145,
      "rewards/margins": 0.0267957653850317,
      "rewards/rejected": -8.235433578491211,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 10.181057969467895,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -1.4492734670639038,
      "logits/rejected": -1.4457294940948486,
      "logps/chosen": -8.069734573364258,
      "logps/rejected": -8.081002235412598,
      "loss": 9.1066,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -8.069734573364258,
      "rewards/margins": 0.011268272995948792,
      "rewards/rejected": -8.081002235412598,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 17.86320335028244,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -1.4542796611785889,
      "logits/rejected": -1.4671558141708374,
      "logps/chosen": -8.027522087097168,
      "logps/rejected": -8.309490203857422,
      "loss": 8.9006,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -8.027522087097168,
      "rewards/margins": 0.28196847438812256,
      "rewards/rejected": -8.309490203857422,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 31.178844889844076,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -1.4044889211654663,
      "logits/rejected": -1.4228571653366089,
      "logps/chosen": -7.809567451477051,
      "logps/rejected": -8.305734634399414,
      "loss": 8.9379,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -7.809567451477051,
      "rewards/margins": 0.49616700410842896,
      "rewards/rejected": -8.305734634399414,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 35.6438890099215,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -1.4016475677490234,
      "logits/rejected": -1.3958280086517334,
      "logps/chosen": -7.9446282386779785,
      "logps/rejected": -8.026212692260742,
      "loss": 8.9738,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -7.9446282386779785,
      "rewards/margins": 0.08158435672521591,
      "rewards/rejected": -8.026212692260742,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 12.429786415191892,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -1.4062070846557617,
      "logits/rejected": -1.412986159324646,
      "logps/chosen": -7.849791049957275,
      "logps/rejected": -7.881753444671631,
      "loss": 8.9235,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -7.849791049957275,
      "rewards/margins": 0.031962424516677856,
      "rewards/rejected": -7.881753444671631,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 19.088641406767135,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -1.4117752313613892,
      "logits/rejected": -1.4267140626907349,
      "logps/chosen": -7.871654510498047,
      "logps/rejected": -7.936394691467285,
      "loss": 8.9114,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -7.871654510498047,
      "rewards/margins": 0.06473957002162933,
      "rewards/rejected": -7.936394691467285,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 12.122887672693283,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -1.4204871654510498,
      "logits/rejected": -1.4174373149871826,
      "logps/chosen": -7.965147495269775,
      "logps/rejected": -8.144342422485352,
      "loss": 9.0544,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -7.965147495269775,
      "rewards/margins": 0.17919424176216125,
      "rewards/rejected": -8.144342422485352,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 13.676489394050206,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -1.394677996635437,
      "logits/rejected": -1.4083014726638794,
      "logps/chosen": -7.955393314361572,
      "logps/rejected": -8.132440567016602,
      "loss": 9.0209,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.955393314361572,
      "rewards/margins": 0.1770474910736084,
      "rewards/rejected": -8.132440567016602,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 11.55232614681137,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -1.4072471857070923,
      "logits/rejected": -1.3971055746078491,
      "logps/chosen": -8.058225631713867,
      "logps/rejected": -7.9064154624938965,
      "loss": 9.0891,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -8.058225631713867,
      "rewards/margins": -0.15181036293506622,
      "rewards/rejected": -7.9064154624938965,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 21.381914106448818,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -1.4291296005249023,
      "logits/rejected": -1.4334924221038818,
      "logps/chosen": -8.016979217529297,
      "logps/rejected": -8.049524307250977,
      "loss": 9.0182,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -8.016979217529297,
      "rewards/margins": 0.032544612884521484,
      "rewards/rejected": -8.049524307250977,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 12.15933543492837,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -1.4222753047943115,
      "logits/rejected": -1.4167351722717285,
      "logps/chosen": -7.735973358154297,
      "logps/rejected": -7.934323310852051,
      "loss": 8.9474,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -7.735973358154297,
      "rewards/margins": 0.19834883511066437,
      "rewards/rejected": -7.934323310852051,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 14.275715375180301,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -1.4484798908233643,
      "logits/rejected": -1.4482091665267944,
      "logps/chosen": -7.996209144592285,
      "logps/rejected": -8.223912239074707,
      "loss": 9.021,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -7.996209144592285,
      "rewards/margins": 0.22770316898822784,
      "rewards/rejected": -8.223912239074707,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 264.49732707332146,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -1.3956806659698486,
      "logits/rejected": -1.399662971496582,
      "logps/chosen": -7.871851921081543,
      "logps/rejected": -8.169795989990234,
      "loss": 8.9644,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -7.871851921081543,
      "rewards/margins": 0.2979447841644287,
      "rewards/rejected": -8.169795989990234,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 13.052538835040968,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -1.3686974048614502,
      "logits/rejected": -1.3719284534454346,
      "logps/chosen": -8.102027893066406,
      "logps/rejected": -8.149332046508789,
      "loss": 8.9524,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.102027893066406,
      "rewards/margins": 0.04730508476495743,
      "rewards/rejected": -8.149332046508789,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 13.986017227821636,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -1.4253513813018799,
      "logits/rejected": -1.419106125831604,
      "logps/chosen": -8.198808670043945,
      "logps/rejected": -8.076632499694824,
      "loss": 9.068,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -8.198808670043945,
      "rewards/margins": -0.12217602878808975,
      "rewards/rejected": -8.076632499694824,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 16.859825952779723,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -1.393610954284668,
      "logits/rejected": -1.4041980504989624,
      "logps/chosen": -8.033707618713379,
      "logps/rejected": -8.149084091186523,
      "loss": 8.9812,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -8.033707618713379,
      "rewards/margins": 0.11537569761276245,
      "rewards/rejected": -8.149084091186523,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 17.35250325551846,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -1.3907815217971802,
      "logits/rejected": -1.4047492742538452,
      "logps/chosen": -7.996786594390869,
      "logps/rejected": -8.4099760055542,
      "loss": 8.9754,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -7.996786594390869,
      "rewards/margins": 0.41318947076797485,
      "rewards/rejected": -8.4099760055542,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 13.984769587358256,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -1.415302038192749,
      "logits/rejected": -1.4232456684112549,
      "logps/chosen": -7.9026665687561035,
      "logps/rejected": -7.970452785491943,
      "loss": 8.9574,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -7.9026665687561035,
      "rewards/margins": 0.06778701394796371,
      "rewards/rejected": -7.970452785491943,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 10.033908137947623,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -1.4233510494232178,
      "logits/rejected": -1.4498116970062256,
      "logps/chosen": -7.896154880523682,
      "logps/rejected": -8.049067497253418,
      "loss": 8.9238,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -7.896154880523682,
      "rewards/margins": 0.15291360020637512,
      "rewards/rejected": -8.049067497253418,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 12.948300110975106,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -1.4413058757781982,
      "logits/rejected": -1.4447827339172363,
      "logps/chosen": -7.901156425476074,
      "logps/rejected": -8.029939651489258,
      "loss": 8.9805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.901156425476074,
      "rewards/margins": 0.12878116965293884,
      "rewards/rejected": -8.029939651489258,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 13.439201394209311,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -1.412276029586792,
      "logits/rejected": -1.389304280281067,
      "logps/chosen": -7.978514194488525,
      "logps/rejected": -7.953900337219238,
      "loss": 9.0293,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -7.978514194488525,
      "rewards/margins": -0.024613410234451294,
      "rewards/rejected": -7.953900337219238,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 14.950263562725501,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -1.3908809423446655,
      "logits/rejected": -1.4011869430541992,
      "logps/chosen": -8.05128288269043,
      "logps/rejected": -8.073431015014648,
      "loss": 9.0342,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -8.05128288269043,
      "rewards/margins": 0.022147323936223984,
      "rewards/rejected": -8.073431015014648,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 11.065140622134825,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -1.4063256978988647,
      "logits/rejected": -1.4003641605377197,
      "logps/chosen": -7.999358177185059,
      "logps/rejected": -8.034425735473633,
      "loss": 9.0115,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -7.999358177185059,
      "rewards/margins": 0.03506825119256973,
      "rewards/rejected": -8.034425735473633,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 12.669233243410716,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -1.3777008056640625,
      "logits/rejected": -1.3843880891799927,
      "logps/chosen": -8.11161994934082,
      "logps/rejected": -8.079643249511719,
      "loss": 9.088,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -8.11161994934082,
      "rewards/margins": -0.03197728842496872,
      "rewards/rejected": -8.079643249511719,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 11.420602500449908,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -1.4002071619033813,
      "logits/rejected": -1.4017866849899292,
      "logps/chosen": -7.9545183181762695,
      "logps/rejected": -7.929112434387207,
      "loss": 9.01,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -7.9545183181762695,
      "rewards/margins": -0.025406653061509132,
      "rewards/rejected": -7.929112434387207,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 15.567371494550915,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -1.379305124282837,
      "logits/rejected": -1.4001214504241943,
      "logps/chosen": -8.02379322052002,
      "logps/rejected": -8.138644218444824,
      "loss": 8.993,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -8.02379322052002,
      "rewards/margins": 0.11485125869512558,
      "rewards/rejected": -8.138644218444824,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 13.26955840349748,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -1.345184564590454,
      "logits/rejected": -1.3495187759399414,
      "logps/chosen": -7.9985671043396,
      "logps/rejected": -8.032160758972168,
      "loss": 8.9564,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -7.9985671043396,
      "rewards/margins": 0.03359389677643776,
      "rewards/rejected": -8.032160758972168,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 12.477711396229129,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -1.3648011684417725,
      "logits/rejected": -1.3609410524368286,
      "logps/chosen": -7.813986301422119,
      "logps/rejected": -7.937358856201172,
      "loss": 8.9559,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -7.813986301422119,
      "rewards/margins": 0.12337237596511841,
      "rewards/rejected": -7.937358856201172,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 14.974853102646863,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -1.340057611465454,
      "logits/rejected": -1.3464308977127075,
      "logps/chosen": -8.168105125427246,
      "logps/rejected": -8.047191619873047,
      "loss": 9.0593,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -8.168105125427246,
      "rewards/margins": -0.12091370671987534,
      "rewards/rejected": -8.047191619873047,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 12.019056641062388,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -1.3949248790740967,
      "logits/rejected": -1.3864567279815674,
      "logps/chosen": -7.904551029205322,
      "logps/rejected": -8.010711669921875,
      "loss": 9.0251,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -7.904551029205322,
      "rewards/margins": 0.10616078227758408,
      "rewards/rejected": -8.010711669921875,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 12.159976530620005,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -1.3519536256790161,
      "logits/rejected": -1.3532142639160156,
      "logps/chosen": -7.96322774887085,
      "logps/rejected": -8.124198913574219,
      "loss": 8.9502,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.96322774887085,
      "rewards/margins": 0.16097018122673035,
      "rewards/rejected": -8.124198913574219,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 27.53866432550454,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -1.3502663373947144,
      "logits/rejected": -1.3595716953277588,
      "logps/chosen": -7.90906286239624,
      "logps/rejected": -8.031661033630371,
      "loss": 8.997,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -7.90906286239624,
      "rewards/margins": 0.12259860336780548,
      "rewards/rejected": -8.031661033630371,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 10.57208181223796,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -1.3545321226119995,
      "logits/rejected": -1.363968849182129,
      "logps/chosen": -7.766638278961182,
      "logps/rejected": -8.186101913452148,
      "loss": 8.9091,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -7.766638278961182,
      "rewards/margins": 0.4194633960723877,
      "rewards/rejected": -8.186101913452148,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 12.898063816305818,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -1.3691070079803467,
      "logits/rejected": -1.3948938846588135,
      "logps/chosen": -7.7049560546875,
      "logps/rejected": -7.837745666503906,
      "loss": 9.0096,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.7049560546875,
      "rewards/margins": 0.13278940320014954,
      "rewards/rejected": -7.837745666503906,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 11.02542077899864,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -1.3727288246154785,
      "logits/rejected": -1.3770244121551514,
      "logps/chosen": -7.688443660736084,
      "logps/rejected": -7.930489540100098,
      "loss": 8.9147,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -7.688443660736084,
      "rewards/margins": 0.24204584956169128,
      "rewards/rejected": -7.930489540100098,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 12.642936387401933,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -1.3643910884857178,
      "logits/rejected": -1.3646974563598633,
      "logps/chosen": -8.087334632873535,
      "logps/rejected": -8.227707862854004,
      "loss": 9.0751,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.087334632873535,
      "rewards/margins": 0.1403738558292389,
      "rewards/rejected": -8.227707862854004,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 11.653678081935743,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -1.3848552703857422,
      "logits/rejected": -1.3937251567840576,
      "logps/chosen": -7.939251899719238,
      "logps/rejected": -8.165653228759766,
      "loss": 8.9934,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -7.939251899719238,
      "rewards/margins": 0.2264014482498169,
      "rewards/rejected": -8.165653228759766,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 20.78939179590344,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -1.3638317584991455,
      "logits/rejected": -1.372854471206665,
      "logps/chosen": -7.975274562835693,
      "logps/rejected": -8.114542007446289,
      "loss": 9.0759,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -7.975274562835693,
      "rewards/margins": 0.13926678895950317,
      "rewards/rejected": -8.114542007446289,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 15.247620089410335,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -1.3632863759994507,
      "logits/rejected": -1.368691086769104,
      "logps/chosen": -8.058801651000977,
      "logps/rejected": -8.10949420928955,
      "loss": 8.9298,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -8.058801651000977,
      "rewards/margins": 0.05069229006767273,
      "rewards/rejected": -8.10949420928955,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 16.020124654557804,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -1.335317611694336,
      "logits/rejected": -1.3454726934432983,
      "logps/chosen": -7.886631011962891,
      "logps/rejected": -8.195113182067871,
      "loss": 8.8247,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -7.886631011962891,
      "rewards/margins": 0.30848345160484314,
      "rewards/rejected": -8.195113182067871,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 13.208964203269911,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -1.3199383020401,
      "logits/rejected": -1.3232190608978271,
      "logps/chosen": -8.123445510864258,
      "logps/rejected": -8.300798416137695,
      "loss": 8.9974,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -8.123445510864258,
      "rewards/margins": 0.17735353112220764,
      "rewards/rejected": -8.300798416137695,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 12.80879122998489,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -1.2541121244430542,
      "logits/rejected": -1.2609522342681885,
      "logps/chosen": -7.773028373718262,
      "logps/rejected": -8.211790084838867,
      "loss": 8.9532,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.773028373718262,
      "rewards/margins": 0.43876129388809204,
      "rewards/rejected": -8.211790084838867,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 11.386114149183353,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -1.2998675107955933,
      "logits/rejected": -1.3152759075164795,
      "logps/chosen": -7.926003932952881,
      "logps/rejected": -8.16883373260498,
      "loss": 8.972,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -7.926003932952881,
      "rewards/margins": 0.242829367518425,
      "rewards/rejected": -8.16883373260498,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 12.189434902359803,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -1.311598539352417,
      "logits/rejected": -1.2913634777069092,
      "logps/chosen": -8.263054847717285,
      "logps/rejected": -8.317811965942383,
      "loss": 9.0177,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -8.263054847717285,
      "rewards/margins": 0.05475734546780586,
      "rewards/rejected": -8.317811965942383,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 26.512104480113177,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -1.3503350019454956,
      "logits/rejected": -1.3630883693695068,
      "logps/chosen": -7.8860344886779785,
      "logps/rejected": -8.053815841674805,
      "loss": 8.9101,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.8860344886779785,
      "rewards/margins": 0.1677820235490799,
      "rewards/rejected": -8.053815841674805,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 13.657892445365238,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -1.3591408729553223,
      "logits/rejected": -1.3795652389526367,
      "logps/chosen": -7.8742218017578125,
      "logps/rejected": -8.096671104431152,
      "loss": 8.926,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -7.8742218017578125,
      "rewards/margins": 0.222448468208313,
      "rewards/rejected": -8.096671104431152,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 13.125973843054254,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -1.3040008544921875,
      "logits/rejected": -1.3099054098129272,
      "logps/chosen": -7.843650817871094,
      "logps/rejected": -8.080728530883789,
      "loss": 8.8759,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -7.843650817871094,
      "rewards/margins": 0.23707744479179382,
      "rewards/rejected": -8.080728530883789,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 12.761067879239661,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -1.3441493511199951,
      "logits/rejected": -1.3643635511398315,
      "logps/chosen": -7.810141086578369,
      "logps/rejected": -8.124038696289062,
      "loss": 8.9111,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -7.810141086578369,
      "rewards/margins": 0.31389790773391724,
      "rewards/rejected": -8.124038696289062,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 12.064219944453757,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -1.3431895971298218,
      "logits/rejected": -1.3602094650268555,
      "logps/chosen": -7.9335832595825195,
      "logps/rejected": -8.049263000488281,
      "loss": 8.9788,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -7.9335832595825195,
      "rewards/margins": 0.11567918956279755,
      "rewards/rejected": -8.049263000488281,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 12.249297326029936,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -1.3094807863235474,
      "logits/rejected": -1.3196088075637817,
      "logps/chosen": -7.965109825134277,
      "logps/rejected": -8.200112342834473,
      "loss": 8.9053,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.965109825134277,
      "rewards/margins": 0.23500177264213562,
      "rewards/rejected": -8.200112342834473,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 11.932717244582381,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -1.317882776260376,
      "logits/rejected": -1.3457939624786377,
      "logps/chosen": -8.174354553222656,
      "logps/rejected": -8.366888046264648,
      "loss": 9.0377,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -8.174354553222656,
      "rewards/margins": 0.19253170490264893,
      "rewards/rejected": -8.366888046264648,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 14.13608210525545,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -1.3378267288208008,
      "logits/rejected": -1.3463207483291626,
      "logps/chosen": -8.089614868164062,
      "logps/rejected": -8.071249008178711,
      "loss": 9.0688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.089614868164062,
      "rewards/margins": -0.018364891409873962,
      "rewards/rejected": -8.071249008178711,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 32.011426029911085,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -1.330702304840088,
      "logits/rejected": -1.3502042293548584,
      "logps/chosen": -7.904356479644775,
      "logps/rejected": -8.121664047241211,
      "loss": 8.8938,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.904356479644775,
      "rewards/margins": 0.2173081338405609,
      "rewards/rejected": -8.121664047241211,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 12.807799082013823,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -1.346669316291809,
      "logits/rejected": -1.3398444652557373,
      "logps/chosen": -8.155746459960938,
      "logps/rejected": -8.042104721069336,
      "loss": 8.9844,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -8.155746459960938,
      "rewards/margins": -0.11364219337701797,
      "rewards/rejected": -8.042104721069336,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 14.721102641186317,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -1.3383146524429321,
      "logits/rejected": -1.3453185558319092,
      "logps/chosen": -8.06794548034668,
      "logps/rejected": -8.342490196228027,
      "loss": 8.9313,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.06794548034668,
      "rewards/margins": 0.27454426884651184,
      "rewards/rejected": -8.342490196228027,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 12.639814201884843,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -1.3189464807510376,
      "logits/rejected": -1.3342128992080688,
      "logps/chosen": -7.980110168457031,
      "logps/rejected": -8.00477123260498,
      "loss": 9.0333,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.980110168457031,
      "rewards/margins": 0.024661889299750328,
      "rewards/rejected": -8.00477123260498,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 13.542292770349258,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -1.3359485864639282,
      "logits/rejected": -1.3414697647094727,
      "logps/chosen": -7.863481044769287,
      "logps/rejected": -8.104010581970215,
      "loss": 8.8907,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.863481044769287,
      "rewards/margins": 0.24052973091602325,
      "rewards/rejected": -8.104010581970215,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 13.235795007023567,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -1.3393312692642212,
      "logits/rejected": -1.3506433963775635,
      "logps/chosen": -8.217998504638672,
      "logps/rejected": -8.318530082702637,
      "loss": 9.0332,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -8.217998504638672,
      "rewards/margins": 0.1005309671163559,
      "rewards/rejected": -8.318530082702637,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 14.926426867380192,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -1.3743078708648682,
      "logits/rejected": -1.390223503112793,
      "logps/chosen": -8.069305419921875,
      "logps/rejected": -8.406010627746582,
      "loss": 9.0149,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.069305419921875,
      "rewards/margins": 0.3367052972316742,
      "rewards/rejected": -8.406010627746582,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 15.925504397841232,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -1.344575047492981,
      "logits/rejected": -1.3378257751464844,
      "logps/chosen": -7.946380615234375,
      "logps/rejected": -8.057405471801758,
      "loss": 8.9081,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -7.946380615234375,
      "rewards/margins": 0.11102403700351715,
      "rewards/rejected": -8.057405471801758,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 14.002913308986452,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -1.3642247915267944,
      "logits/rejected": -1.356299638748169,
      "logps/chosen": -8.019064903259277,
      "logps/rejected": -7.9631476402282715,
      "loss": 8.911,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -8.019064903259277,
      "rewards/margins": -0.055917102843523026,
      "rewards/rejected": -7.9631476402282715,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 11.189613386087895,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -1.323051929473877,
      "logits/rejected": -1.3336186408996582,
      "logps/chosen": -7.749169826507568,
      "logps/rejected": -8.018518447875977,
      "loss": 8.86,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -7.749169826507568,
      "rewards/margins": 0.26934924721717834,
      "rewards/rejected": -8.018518447875977,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 12.73495657042311,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -1.299702525138855,
      "logits/rejected": -1.300682783126831,
      "logps/chosen": -7.912903785705566,
      "logps/rejected": -7.9577813148498535,
      "loss": 8.9548,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.912903785705566,
      "rewards/margins": 0.044878166168928146,
      "rewards/rejected": -7.9577813148498535,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 12.244582800127791,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -1.3569344282150269,
      "logits/rejected": -1.3466898202896118,
      "logps/chosen": -8.036967277526855,
      "logps/rejected": -8.161175727844238,
      "loss": 8.9693,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -8.036967277526855,
      "rewards/margins": 0.12420801818370819,
      "rewards/rejected": -8.161175727844238,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 13.857374473897332,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -1.3418371677398682,
      "logits/rejected": -1.3312771320343018,
      "logps/chosen": -8.015717506408691,
      "logps/rejected": -8.231267929077148,
      "loss": 8.9495,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -8.015717506408691,
      "rewards/margins": 0.2155502289533615,
      "rewards/rejected": -8.231267929077148,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 12.96483071279786,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -1.3451616764068604,
      "logits/rejected": -1.2988903522491455,
      "logps/chosen": -8.223435401916504,
      "logps/rejected": -8.359044075012207,
      "loss": 8.9575,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -8.223435401916504,
      "rewards/margins": 0.13560748100280762,
      "rewards/rejected": -8.359044075012207,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 13.685503251086777,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -1.3425824642181396,
      "logits/rejected": -1.3416216373443604,
      "logps/chosen": -8.126443862915039,
      "logps/rejected": -8.207201957702637,
      "loss": 8.9674,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -8.126443862915039,
      "rewards/margins": 0.08075866848230362,
      "rewards/rejected": -8.207201957702637,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 13.422439706317645,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -1.3507561683654785,
      "logits/rejected": -1.3538233041763306,
      "logps/chosen": -7.850442409515381,
      "logps/rejected": -8.027676582336426,
      "loss": 8.8745,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -7.850442409515381,
      "rewards/margins": 0.17723475396633148,
      "rewards/rejected": -8.027676582336426,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 17.927875137352093,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -1.3154563903808594,
      "logits/rejected": -1.3293858766555786,
      "logps/chosen": -7.652901649475098,
      "logps/rejected": -7.787165641784668,
      "loss": 8.9212,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.652901649475098,
      "rewards/margins": 0.13426539301872253,
      "rewards/rejected": -7.787165641784668,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 15.347923517709377,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -1.3166577816009521,
      "logits/rejected": -1.308420181274414,
      "logps/chosen": -7.855776786804199,
      "logps/rejected": -7.910757541656494,
      "loss": 8.958,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -7.855776786804199,
      "rewards/margins": 0.05497948080301285,
      "rewards/rejected": -7.910757541656494,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 12.360735869108282,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -1.3554697036743164,
      "logits/rejected": -1.361022710800171,
      "logps/chosen": -8.025907516479492,
      "logps/rejected": -8.157610893249512,
      "loss": 8.9282,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -8.025907516479492,
      "rewards/margins": 0.13170403242111206,
      "rewards/rejected": -8.157610893249512,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 31.497593173498622,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -1.3601171970367432,
      "logits/rejected": -1.3568463325500488,
      "logps/chosen": -7.857785701751709,
      "logps/rejected": -7.846571922302246,
      "loss": 8.9122,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.857785701751709,
      "rewards/margins": -0.01121367234736681,
      "rewards/rejected": -7.846571922302246,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 14.186881098656178,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -1.3695319890975952,
      "logits/rejected": -1.3772709369659424,
      "logps/chosen": -7.996212959289551,
      "logps/rejected": -8.215394973754883,
      "loss": 8.984,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -7.996212959289551,
      "rewards/margins": 0.21918316185474396,
      "rewards/rejected": -8.215394973754883,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 13.07091749141306,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -1.320655107498169,
      "logits/rejected": -1.315647006034851,
      "logps/chosen": -7.8787431716918945,
      "logps/rejected": -7.92669677734375,
      "loss": 8.9426,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -7.8787431716918945,
      "rewards/margins": 0.047954294830560684,
      "rewards/rejected": -7.92669677734375,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 12.735545462610197,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -1.3389335870742798,
      "logits/rejected": -1.335252285003662,
      "logps/chosen": -8.04029655456543,
      "logps/rejected": -8.07039737701416,
      "loss": 8.9855,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -8.04029655456543,
      "rewards/margins": 0.030100971460342407,
      "rewards/rejected": -8.07039737701416,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 15.247676128020851,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -1.3410335779190063,
      "logits/rejected": -1.3450171947479248,
      "logps/chosen": -7.889002323150635,
      "logps/rejected": -7.991044044494629,
      "loss": 8.9385,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -7.889002323150635,
      "rewards/margins": 0.10204191505908966,
      "rewards/rejected": -7.991044044494629,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 13.85977210657239,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -1.3579423427581787,
      "logits/rejected": -1.3657190799713135,
      "logps/chosen": -7.948614597320557,
      "logps/rejected": -8.05911636352539,
      "loss": 8.8473,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.948614597320557,
      "rewards/margins": 0.11050298064947128,
      "rewards/rejected": -8.05911636352539,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 13.136814821473001,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -1.327292799949646,
      "logits/rejected": -1.3260419368743896,
      "logps/chosen": -8.011943817138672,
      "logps/rejected": -8.084588050842285,
      "loss": 9.0087,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -8.011943817138672,
      "rewards/margins": 0.07264318317174911,
      "rewards/rejected": -8.084588050842285,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 13.772252146186085,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -1.3517519235610962,
      "logits/rejected": -1.353088140487671,
      "logps/chosen": -7.9066948890686035,
      "logps/rejected": -8.016542434692383,
      "loss": 8.9143,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -7.9066948890686035,
      "rewards/margins": 0.10984818637371063,
      "rewards/rejected": -8.016542434692383,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 62.09067243328178,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -1.3028905391693115,
      "logits/rejected": -1.2930048704147339,
      "logps/chosen": -8.025864601135254,
      "logps/rejected": -8.012077331542969,
      "loss": 9.0079,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -8.025864601135254,
      "rewards/margins": -0.013786206021904945,
      "rewards/rejected": -8.012077331542969,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 13.419716763500347,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -1.3575193881988525,
      "logits/rejected": -1.351488709449768,
      "logps/chosen": -7.829765319824219,
      "logps/rejected": -8.183721542358398,
      "loss": 8.9117,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -7.829765319824219,
      "rewards/margins": 0.3539566695690155,
      "rewards/rejected": -8.183721542358398,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 22.929649885843165,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -1.3399038314819336,
      "logits/rejected": -1.352718710899353,
      "logps/chosen": -7.996813774108887,
      "logps/rejected": -8.18346881866455,
      "loss": 8.8927,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.996813774108887,
      "rewards/margins": 0.18665429949760437,
      "rewards/rejected": -8.18346881866455,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 12.997334149917643,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -1.363525390625,
      "logits/rejected": -1.3654100894927979,
      "logps/chosen": -8.123039245605469,
      "logps/rejected": -8.254613876342773,
      "loss": 8.999,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -8.123039245605469,
      "rewards/margins": 0.13157425820827484,
      "rewards/rejected": -8.254613876342773,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 14.345805639596328,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -1.3454748392105103,
      "logits/rejected": -1.3637425899505615,
      "logps/chosen": -7.719171047210693,
      "logps/rejected": -7.984010219573975,
      "loss": 8.9911,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -7.719171047210693,
      "rewards/margins": 0.26483961939811707,
      "rewards/rejected": -7.984010219573975,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 15.884627478532495,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -1.3404934406280518,
      "logits/rejected": -1.329254388809204,
      "logps/chosen": -8.111069679260254,
      "logps/rejected": -8.152128219604492,
      "loss": 8.9896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -8.111069679260254,
      "rewards/margins": 0.041059140115976334,
      "rewards/rejected": -8.152128219604492,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 8.9737869758526,
      "train_runtime": 8198.1649,
      "train_samples_per_second": 7.457,
      "train_steps_per_second": 0.058
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}