{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 100,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.3086,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8665881156921387,
      "logits/rejected": -1.8709055185317993,
      "logps/chosen": -36.99662399291992,
      "logps/rejected": -33.65571594238281,
      "loss": 0.3052,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.008980684913694859,
      "rewards/margins": 0.032059140503406525,
      "rewards/rejected": -0.02307845838367939,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9977455139160156,
      "logits/rejected": -2.000382423400879,
      "logps/chosen": -29.642925262451172,
      "logps/rejected": -29.056737899780273,
      "loss": 0.3525,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0006573178106918931,
      "rewards/margins": -0.00669272243976593,
      "rewards/rejected": 0.006035405211150646,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9204607009887695,
      "logits/rejected": -1.9177772998809814,
      "logps/chosen": -31.42336654663086,
      "logps/rejected": -33.22785568237305,
      "loss": 0.3603,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.006546213291585445,
      "rewards/margins": 0.0019128695130348206,
      "rewards/rejected": -0.00845908559858799,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0172362327575684,
      "logits/rejected": -2.008507251739502,
      "logps/chosen": -32.56964874267578,
      "logps/rejected": -32.50572967529297,
      "loss": 0.3558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006439635064452887,
      "rewards/margins": 0.00019515231542754918,
      "rewards/rejected": 0.006244482938200235,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8618619441986084,
      "logits/rejected": -1.8510783910751343,
      "logps/chosen": -33.56026077270508,
      "logps/rejected": -35.45254898071289,
      "loss": 0.3691,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0025013976264744997,
      "rewards/margins": -0.002315213903784752,
      "rewards/rejected": -0.00018618404283188283,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9393202066421509,
      "logits/rejected": -1.9412600994110107,
      "logps/chosen": -32.57838439941406,
      "logps/rejected": -33.215576171875,
      "loss": 0.3063,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.01962057128548622,
      "rewards/margins": 0.05940054729580879,
      "rewards/rejected": -0.03977997973561287,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.0718436241149902,
      "logits/rejected": -2.0768017768859863,
      "logps/chosen": -33.97806167602539,
      "logps/rejected": -36.63082504272461,
      "loss": 0.4257,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00938049890100956,
      "rewards/margins": 0.06513925641775131,
      "rewards/rejected": -0.0557587556540966,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9351739883422852,
      "logits/rejected": -1.9383188486099243,
      "logps/chosen": -34.33073043823242,
      "logps/rejected": -34.61904525756836,
      "loss": 0.2902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08736880123615265,
      "rewards/margins": 0.11134655773639679,
      "rewards/rejected": -0.023977745324373245,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.944392204284668,
      "logits/rejected": -1.9489190578460693,
      "logps/chosen": -32.419586181640625,
      "logps/rejected": -32.3698844909668,
      "loss": 0.3611,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.043051257729530334,
      "rewards/margins": 0.052930813282728195,
      "rewards/rejected": -0.009879561141133308,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.0419769287109375,
      "logits/rejected": -2.0399627685546875,
      "logps/chosen": -32.174407958984375,
      "logps/rejected": -31.26608657836914,
      "loss": 0.3038,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.059785228222608566,
      "rewards/margins": 0.05967242643237114,
      "rewards/rejected": 0.00011279433965682983,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2372143268585205,
      "eval_logits/rejected": -2.2323503494262695,
      "eval_logps/chosen": -34.01276779174805,
      "eval_logps/rejected": -37.51152420043945,
      "eval_loss": 0.3511974811553955,
      "eval_rewards/accuracies": 0.5423588156700134,
      "eval_rewards/chosen": 0.01960929110646248,
      "eval_rewards/margins": 0.01502405758947134,
      "eval_rewards/rejected": 0.004585230257362127,
      "eval_runtime": 145.9032,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.998891830444336,
      "logits/rejected": -1.996492624282837,
      "logps/chosen": -33.14598083496094,
      "logps/rejected": -34.020729064941406,
      "loss": 0.4689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.08805312216281891,
      "rewards/margins": 0.08027410507202148,
      "rewards/rejected": 0.007779018487781286,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.010932445526123,
      "logits/rejected": -2.0025696754455566,
      "logps/chosen": -32.37172317504883,
      "logps/rejected": -32.118797302246094,
      "loss": 0.4465,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.06586603820323944,
      "rewards/margins": 0.02742874063551426,
      "rewards/rejected": 0.03843729570508003,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0387539863586426,
      "logits/rejected": -2.030724287033081,
      "logps/chosen": -30.41655921936035,
      "logps/rejected": -32.060333251953125,
      "loss": 0.3844,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.05324209854006767,
      "rewards/margins": 0.05983499437570572,
      "rewards/rejected": -0.00659290561452508,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.967858076095581,
      "logits/rejected": -1.9781148433685303,
      "logps/chosen": -31.223413467407227,
      "logps/rejected": -32.55517578125,
      "loss": 0.3904,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1766217201948166,
      "rewards/margins": 0.1948881596326828,
      "rewards/rejected": -0.01826643943786621,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.880910873413086,
      "logits/rejected": -1.8820507526397705,
      "logps/chosen": -34.01464080810547,
      "logps/rejected": -34.783546447753906,
      "loss": 0.3871,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.160243421792984,
      "rewards/margins": 0.1839032918214798,
      "rewards/rejected": -0.023659853264689445,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.933895468711853,
      "logits/rejected": -1.9304730892181396,
      "logps/chosen": -36.02853775024414,
      "logps/rejected": -32.699058532714844,
      "loss": 0.2729,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1288261115550995,
      "rewards/margins": 0.10750452429056168,
      "rewards/rejected": 0.021321602165699005,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0341715812683105,
      "logits/rejected": -2.0267820358276367,
      "logps/chosen": -33.55347442626953,
      "logps/rejected": -31.3526554107666,
      "loss": 0.294,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.21994857490062714,
      "rewards/margins": 0.22067300975322723,
      "rewards/rejected": -0.0007244400912895799,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0400891304016113,
      "logits/rejected": -2.045360565185547,
      "logps/chosen": -32.370338439941406,
      "logps/rejected": -32.4719123840332,
      "loss": 0.2829,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.15700635313987732,
      "rewards/margins": 0.145101398229599,
      "rewards/rejected": 0.011904975399374962,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.041393280029297,
      "logits/rejected": -2.038623809814453,
      "logps/chosen": -31.328174591064453,
      "logps/rejected": -31.316492080688477,
      "loss": 0.3044,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.14470075070858002,
      "rewards/margins": 0.1431477963924408,
      "rewards/rejected": 0.0015529401134699583,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9122215509414673,
      "logits/rejected": -1.9168663024902344,
      "logps/chosen": -31.424020767211914,
      "logps/rejected": -32.784080505371094,
      "loss": 0.3157,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.16403506696224213,
      "rewards/margins": 0.1578713059425354,
      "rewards/rejected": 0.0061637843027710915,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.237051010131836,
      "eval_logits/rejected": -2.2321863174438477,
      "eval_logps/chosen": -34.018070220947266,
      "eval_logps/rejected": -37.51838684082031,
      "eval_loss": 0.3716273605823517,
      "eval_rewards/accuracies": 0.5245016813278198,
      "eval_rewards/chosen": 0.014834923669695854,
      "eval_rewards/margins": 0.016425320878624916,
      "eval_rewards/rejected": -0.0015903981402516365,
      "eval_runtime": 145.5151,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.296,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.023789882659912,
      "logits/rejected": -2.034484386444092,
      "logps/chosen": -31.767370223999023,
      "logps/rejected": -33.890621185302734,
      "loss": 0.275,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.18354059755802155,
      "rewards/margins": 0.20867136120796204,
      "rewards/rejected": -0.025130782276391983,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.916685700416565,
      "logits/rejected": -1.9314892292022705,
      "logps/chosen": -29.956628799438477,
      "logps/rejected": -31.564035415649414,
      "loss": 0.2958,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13942097127437592,
      "rewards/margins": 0.14469322562217712,
      "rewards/rejected": -0.005272268317639828,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9737945795059204,
      "logits/rejected": -1.9777710437774658,
      "logps/chosen": -33.19129180908203,
      "logps/rejected": -31.5566463470459,
      "loss": 0.287,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.20355579257011414,
      "rewards/margins": 0.21016716957092285,
      "rewards/rejected": -0.0066113718785345554,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9743419885635376,
      "logits/rejected": -1.9523779153823853,
      "logps/chosen": -33.9401969909668,
      "logps/rejected": -35.008758544921875,
      "loss": 0.3185,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1921955645084381,
      "rewards/margins": 0.24436470866203308,
      "rewards/rejected": -0.052169155329465866,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0161709785461426,
      "logits/rejected": -2.0128414630889893,
      "logps/chosen": -32.74829864501953,
      "logps/rejected": -36.236392974853516,
      "loss": 0.2847,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.15909257531166077,
      "rewards/margins": 0.1694144755601883,
      "rewards/rejected": -0.010321905836462975,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8847742080688477,
      "logits/rejected": -1.8823268413543701,
      "logps/chosen": -34.01182174682617,
      "logps/rejected": -35.481346130371094,
      "loss": 0.3073,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.1589193046092987,
      "rewards/margins": 0.13797220587730408,
      "rewards/rejected": 0.020947108045220375,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8693536520004272,
      "logits/rejected": -1.866838812828064,
      "logps/chosen": -34.20549392700195,
      "logps/rejected": -31.726673126220703,
      "loss": 0.2733,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.17289286851882935,
      "rewards/margins": 0.1327010840177536,
      "rewards/rejected": 0.040191780775785446,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9730831384658813,
      "logits/rejected": -1.962480902671814,
      "logps/chosen": -35.06049728393555,
      "logps/rejected": -31.792781829833984,
      "loss": 0.2523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2665289342403412,
      "rewards/margins": 0.22314274311065674,
      "rewards/rejected": 0.04338619112968445,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0680813789367676,
      "logits/rejected": -2.053079128265381,
      "logps/chosen": -30.738479614257812,
      "logps/rejected": -32.61243438720703,
      "loss": 0.3194,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.16707859933376312,
      "rewards/margins": 0.13161785900592804,
      "rewards/rejected": 0.035460732877254486,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9398882389068604,
      "logits/rejected": -1.9373395442962646,
      "logps/chosen": -32.6181640625,
      "logps/rejected": -30.843700408935547,
      "loss": 0.2156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2780519425868988,
      "rewards/margins": 0.3022567331790924,
      "rewards/rejected": -0.024204757064580917,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2364187240600586,
      "eval_logits/rejected": -2.2315518856048584,
      "eval_logps/chosen": -34.01428985595703,
      "eval_logps/rejected": -37.496952056884766,
      "eval_loss": 0.38450533151626587,
      "eval_rewards/accuracies": 0.49335551261901855,
      "eval_rewards/chosen": 0.018236981704831123,
      "eval_rewards/margins": 0.0005384809919632971,
      "eval_rewards/rejected": 0.01769850216805935,
      "eval_runtime": 145.7485,
      "eval_samples_per_second": 2.353,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 107.0,
      "learning_rate": 4.84533120650964e-06,
      "logits/chosen": -2.0731887817382812,
      "logits/rejected": -2.060344934463501,
      "logps/chosen": -32.24032211303711,
      "logps/rejected": -32.77452087402344,
      "loss": 0.2321,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.2592628598213196,
      "rewards/margins": 0.3793802857398987,
      "rewards/rejected": -0.12011740356683731,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 37.0,
      "learning_rate": 4.825108134172131e-06,
      "logits/chosen": -1.9875545501708984,
      "logits/rejected": -1.978822112083435,
      "logps/chosen": -32.03451919555664,
      "logps/rejected": -30.3249454498291,
      "loss": 0.2062,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.288422554731369,
      "rewards/margins": 0.3636184334754944,
      "rewards/rejected": -0.07519589364528656,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 25.75,
      "learning_rate": 4.80369052967602e-06,
      "logits/chosen": -1.9262025356292725,
      "logits/rejected": -1.9383175373077393,
      "logps/chosen": -30.169696807861328,
      "logps/rejected": -33.530517578125,
      "loss": 0.214,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.2786136865615845,
      "rewards/margins": 0.3940487504005432,
      "rewards/rejected": -0.11543502658605576,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 52.75,
      "learning_rate": 4.781089396387968e-06,
      "logits/chosen": -1.8931338787078857,
      "logits/rejected": -1.8838812112808228,
      "logps/chosen": -34.24372100830078,
      "logps/rejected": -35.92654037475586,
      "loss": 0.3165,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.40968865156173706,
      "rewards/margins": 0.47252458333969116,
      "rewards/rejected": -0.06283595412969589,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 38.75,
      "learning_rate": 4.757316345716554e-06,
      "logits/chosen": -1.9516513347625732,
      "logits/rejected": -1.9522939920425415,
      "logps/chosen": -34.0687141418457,
      "logps/rejected": -33.91803741455078,
      "loss": 0.2699,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.30291253328323364,
      "rewards/margins": 0.3650719225406647,
      "rewards/rejected": -0.06215935945510864,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 44.75,
      "learning_rate": 4.73238359114687e-06,
      "logits/chosen": -2.07902193069458,
      "logits/rejected": -2.085242748260498,
      "logps/chosen": -31.320343017578125,
      "logps/rejected": -32.6987419128418,
      "loss": 0.3034,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.22761543095111847,
      "rewards/margins": 0.2550276219844818,
      "rewards/rejected": -0.027412226423621178,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 94.0,
      "learning_rate": 4.706303941965804e-06,
      "logits/chosen": -2.0127124786376953,
      "logits/rejected": -2.0121982097625732,
      "logps/chosen": -33.03043746948242,
      "logps/rejected": -36.05421447753906,
      "loss": 0.294,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.3730550706386566,
      "rewards/margins": 0.3770460784435272,
      "rewards/rejected": -0.003990952856838703,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 27.25,
      "learning_rate": 4.679090796681225e-06,
      "logits/chosen": -2.0482752323150635,
      "logits/rejected": -2.043614625930786,
      "logps/chosen": -30.266733169555664,
      "logps/rejected": -29.273147583007812,
      "loss": 0.3277,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3994811177253723,
      "rewards/margins": 0.45281118154525757,
      "rewards/rejected": -0.05333010479807854,
      "step": 380
    },
    {
      "epoch": 1.01,
      "grad_norm": 48.75,
      "learning_rate": 4.650758136138454e-06,
      "logits/chosen": -1.8216171264648438,
      "logits/rejected": -1.8280115127563477,
      "logps/chosen": -32.05220031738281,
      "logps/rejected": -36.293907165527344,
      "loss": 0.4405,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.30016615986824036,
      "rewards/margins": 0.45822396874427795,
      "rewards/rejected": -0.1580578237771988,
      "step": 390
    },
    {
      "epoch": 1.04,
      "grad_norm": 60.75,
      "learning_rate": 4.621320516337559e-06,
      "logits/chosen": -1.9820178747177124,
      "logits/rejected": -1.9757049083709717,
      "logps/chosen": -33.380653381347656,
      "logps/rejected": -32.34453582763672,
      "loss": 0.4084,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.4068370759487152,
      "rewards/margins": 0.43893057107925415,
      "rewards/rejected": -0.032093439251184464,
      "step": 400
    },
    {
      "epoch": 1.04,
      "eval_logits/chosen": -2.244844436645508,
      "eval_logits/rejected": -2.2399790287017822,
      "eval_logps/chosen": -33.95623016357422,
      "eval_logps/rejected": -37.43685531616211,
      "eval_loss": 0.40588921308517456,
      "eval_rewards/accuracies": 0.5066444873809814,
      "eval_rewards/chosen": 0.07048822194337845,
      "eval_rewards/margins": -0.001299927243962884,
      "eval_rewards/rejected": 0.07178815454244614,
      "eval_runtime": 145.5279,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.295,
      "step": 400
    },
    {
      "epoch": 1.06,
      "grad_norm": 37.5,
      "learning_rate": 4.590793060955158e-06,
      "logits/chosen": -1.981856107711792,
      "logits/rejected": -1.9890705347061157,
      "logps/chosen": -28.692569732666016,
      "logps/rejected": -29.251338958740234,
      "loss": 0.3376,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3850264847278595,
      "rewards/margins": 0.5228284597396851,
      "rewards/rejected": -0.13780196011066437,
      "step": 410
    },
    {
      "epoch": 1.09,
      "grad_norm": 46.5,
      "learning_rate": 4.559191453574582e-06,
      "logits/chosen": -2.0013296604156494,
      "logits/rejected": -2.000218152999878,
      "logps/chosen": -33.88956832885742,
      "logps/rejected": -30.98602294921875,
      "loss": 0.244,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.32502850890159607,
      "rewards/margins": 0.36370736360549927,
      "rewards/rejected": -0.038678836077451706,
      "step": 420
    },
    {
      "epoch": 1.12,
      "grad_norm": 74.5,
      "learning_rate": 4.52653192962838e-06,
      "logits/chosen": -1.9944326877593994,
      "logits/rejected": -1.9771124124526978,
      "logps/chosen": -30.677692413330078,
      "logps/rejected": -33.02484893798828,
      "loss": 0.2988,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.269562691450119,
      "rewards/margins": 0.35826626420021057,
      "rewards/rejected": -0.08870354294776917,
      "step": 430
    },
    {
      "epoch": 1.14,
      "grad_norm": 38.5,
      "learning_rate": 4.492831268057307e-06,
      "logits/chosen": -2.0252749919891357,
      "logits/rejected": -2.0271475315093994,
      "logps/chosen": -36.258460998535156,
      "logps/rejected": -34.926544189453125,
      "loss": 0.292,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.31851616501808167,
      "rewards/margins": 0.3784351646900177,
      "rewards/rejected": -0.059918999671936035,
      "step": 440
    },
    {
      "epoch": 1.17,
      "grad_norm": 37.5,
      "learning_rate": 4.458106782690094e-06,
      "logits/chosen": -2.1034398078918457,
      "logits/rejected": -2.1033284664154053,
      "logps/chosen": -32.266780853271484,
      "logps/rejected": -33.54703140258789,
      "loss": 0.5497,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.3299091160297394,
      "rewards/margins": 0.31813210248947144,
      "rewards/rejected": 0.011777004227042198,
      "step": 450
    },
    {
      "epoch": 1.19,
      "grad_norm": 33.75,
      "learning_rate": 4.422376313348405e-06,
      "logits/chosen": -2.0468077659606934,
      "logits/rejected": -2.0393521785736084,
      "logps/chosen": -32.022674560546875,
      "logps/rejected": -36.174320220947266,
      "loss": 0.4905,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.34495481848716736,
      "rewards/margins": 0.4532076418399811,
      "rewards/rejected": -0.10825282335281372,
      "step": 460
    },
    {
      "epoch": 1.22,
      "grad_norm": 59.25,
      "learning_rate": 4.3856582166815696e-06,
      "logits/chosen": -1.9485485553741455,
      "logits/rejected": -1.944986343383789,
      "logps/chosen": -33.546749114990234,
      "logps/rejected": -33.010704040527344,
      "loss": 0.283,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.34699487686157227,
      "rewards/margins": 0.4251338541507721,
      "rewards/rejected": -0.07813899219036102,
      "step": 470
    },
    {
      "epoch": 1.25,
      "grad_norm": 61.25,
      "learning_rate": 4.347971356735789e-06,
      "logits/chosen": -2.071899652481079,
      "logits/rejected": -2.064889907836914,
      "logps/chosen": -30.4763240814209,
      "logps/rejected": -32.066490173339844,
      "loss": 0.2764,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.316992849111557,
      "rewards/margins": 0.3826924264431,
      "rewards/rejected": -0.06569959223270416,
      "step": 480
    },
    {
      "epoch": 1.27,
      "grad_norm": 69.0,
      "learning_rate": 4.309335095262675e-06,
      "logits/chosen": -2.01814341545105,
      "logits/rejected": -2.01973557472229,
      "logps/chosen": -34.93284225463867,
      "logps/rejected": -33.683841705322266,
      "loss": 0.2944,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.37830987572669983,
      "rewards/margins": 0.4164048731327057,
      "rewards/rejected": -0.038094960153102875,
      "step": 490
    },
    {
      "epoch": 1.3,
      "grad_norm": 44.0,
      "learning_rate": 4.269769281772082e-06,
      "logits/chosen": -1.9071893692016602,
      "logits/rejected": -1.9048168659210205,
      "logps/chosen": -32.796241760253906,
      "logps/rejected": -36.574031829833984,
      "loss": 0.2788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33844107389450073,
      "rewards/margins": 0.4371793270111084,
      "rewards/rejected": -0.09873829782009125,
      "step": 500
    },
    {
      "epoch": 1.3,
      "eval_logits/chosen": -2.2405266761779785,
      "eval_logits/rejected": -2.2356326580047607,
      "eval_logps/chosen": -33.95664978027344,
      "eval_logps/rejected": -37.45263671875,
      "eval_loss": 0.38657647371292114,
      "eval_rewards/accuracies": 0.5191029906272888,
      "eval_rewards/chosen": 0.07010918855667114,
      "eval_rewards/margins": 0.012523897923529148,
      "eval_rewards/rejected": 0.05758529528975487,
      "eval_runtime": 145.2533,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.296,
      "step": 500
    },
    {
      "epoch": 1.32,
      "grad_norm": 37.5,
      "learning_rate": 4.22929424333435e-06,
      "logits/chosen": -2.013305187225342,
      "logits/rejected": -2.018036127090454,
      "logps/chosen": -33.06889724731445,
      "logps/rejected": -31.70903968811035,
      "loss": 0.3894,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.34961196780204773,
      "rewards/margins": 0.40624913573265076,
      "rewards/rejected": -0.05663713067770004,
      "step": 510
    },
    {
      "epoch": 1.35,
      "grad_norm": 43.5,
      "learning_rate": 4.1879307741372085e-06,
      "logits/chosen": -2.0473179817199707,
      "logits/rejected": -2.0582363605499268,
      "logps/chosen": -30.8958797454834,
      "logps/rejected": -31.604564666748047,
      "loss": 0.359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44507941603660583,
      "rewards/margins": 0.5322669744491577,
      "rewards/rejected": -0.08718754351139069,
      "step": 520
    },
    {
      "epoch": 1.38,
      "grad_norm": 49.5,
      "learning_rate": 4.145700124802693e-06,
      "logits/chosen": -1.974761962890625,
      "logits/rejected": -1.9713010787963867,
      "logps/chosen": -32.03337097167969,
      "logps/rejected": -32.46263885498047,
      "loss": 0.2845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34842735528945923,
      "rewards/margins": 0.4383048117160797,
      "rewards/rejected": -0.08987746387720108,
      "step": 530
    },
    {
      "epoch": 1.4,
      "grad_norm": 56.0,
      "learning_rate": 4.102623991469562e-06,
      "logits/chosen": -1.8440120220184326,
      "logits/rejected": -1.8531720638275146,
      "logps/chosen": -32.23934555053711,
      "logps/rejected": -31.933460235595703,
      "loss": 0.2635,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.29704010486602783,
      "rewards/margins": 0.36676424741744995,
      "rewards/rejected": -0.06972410529851913,
      "step": 540
    },
    {
      "epoch": 1.43,
      "grad_norm": 41.75,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.9423996210098267,
      "logits/rejected": -1.9359699487686157,
      "logps/chosen": -33.36035919189453,
      "logps/rejected": -31.000152587890625,
      "loss": 0.3772,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.3131468892097473,
      "rewards/margins": 0.32107049226760864,
      "rewards/rejected": -0.007923613302409649,
      "step": 550
    },
    {
      "epoch": 1.45,
      "grad_norm": 147.0,
      "learning_rate": 4.014024217844167e-06,
      "logits/chosen": -2.0321927070617676,
      "logits/rejected": -2.030066967010498,
      "logps/chosen": -34.12587356567383,
      "logps/rejected": -31.518299102783203,
      "loss": 0.5612,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.40079349279403687,
      "rewards/margins": 0.41961637139320374,
      "rewards/rejected": -0.01882282830774784,
      "step": 560
    },
    {
      "epoch": 1.48,
      "grad_norm": 28.125,
      "learning_rate": 3.968546095984911e-06,
      "logits/chosen": -1.8660882711410522,
      "logits/rejected": -1.863891363143921,
      "logps/chosen": -32.46380615234375,
      "logps/rejected": -31.035659790039062,
      "loss": 0.3042,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.2993435263633728,
      "rewards/margins": 0.3293627202510834,
      "rewards/rejected": -0.030019233003258705,
      "step": 570
    },
    {
      "epoch": 1.51,
      "grad_norm": 126.0,
      "learning_rate": 3.922313503607806e-06,
      "logits/chosen": -2.0015358924865723,
      "logits/rejected": -1.998284935951233,
      "logps/chosen": -30.789913177490234,
      "logps/rejected": -34.707244873046875,
      "loss": 0.3323,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.2832598090171814,
      "rewards/margins": 0.3562392592430115,
      "rewards/rejected": -0.07297946512699127,
      "step": 580
    },
    {
      "epoch": 1.53,
      "grad_norm": 26.125,
      "learning_rate": 3.875350192863368e-06,
      "logits/chosen": -1.946082353591919,
      "logits/rejected": -1.9497697353363037,
      "logps/chosen": -29.2518310546875,
      "logps/rejected": -30.736743927001953,
      "loss": 0.3063,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.3268837332725525,
      "rewards/margins": 0.3969722092151642,
      "rewards/rejected": -0.07008843123912811,
      "step": 590
    },
    {
      "epoch": 1.56,
      "grad_norm": 46.25,
      "learning_rate": 3.8276802913111436e-06,
      "logits/chosen": -1.9876987934112549,
      "logits/rejected": -1.9874299764633179,
      "logps/chosen": -31.799388885498047,
      "logps/rejected": -31.247623443603516,
      "loss": 0.3874,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3799284100532532,
      "rewards/margins": 0.4290773868560791,
      "rewards/rejected": -0.049148980528116226,
      "step": 600
    },
    {
      "epoch": 1.56,
      "eval_logits/chosen": -2.2469940185546875,
      "eval_logits/rejected": -2.2421164512634277,
      "eval_logps/chosen": -33.955604553222656,
      "eval_logps/rejected": -37.417686462402344,
      "eval_loss": 0.426466703414917,
      "eval_rewards/accuracies": 0.472591370344162,
      "eval_rewards/chosen": 0.07105044275522232,
      "eval_rewards/margins": -0.017991825938224792,
      "eval_rewards/rejected": 0.08904226124286652,
      "eval_runtime": 145.3878,
      "eval_samples_per_second": 2.359,
      "eval_steps_per_second": 0.296,
      "step": 600
    },
    {
      "epoch": 1.58,
      "grad_norm": 23.5,
      "learning_rate": 3.7793282895240927e-06,
      "logits/chosen": -2.0506019592285156,
      "logits/rejected": -2.051326274871826,
      "logps/chosen": -34.585548400878906,
      "logps/rejected": -33.11695098876953,
      "loss": 0.2877,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.4001394212245941,
      "rewards/margins": 0.4296254515647888,
      "rewards/rejected": -0.02948596142232418,
      "step": 610
    },
    {
      "epoch": 1.61,
      "grad_norm": 69.5,
      "learning_rate": 3.730319028506478e-06,
      "logits/chosen": -2.013078212738037,
      "logits/rejected": -2.010453701019287,
      "logps/chosen": -32.696006774902344,
      "logps/rejected": -32.14519500732422,
      "loss": 0.3336,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.3864508867263794,
      "rewards/margins": 0.4542017877101898,
      "rewards/rejected": -0.06775090843439102,
      "step": 620
    },
    {
      "epoch": 1.64,
      "grad_norm": 31.875,
      "learning_rate": 3.6806776869317074e-06,
      "logits/chosen": -2.02937650680542,
      "logits/rejected": -2.0202343463897705,
      "logps/chosen": -32.354148864746094,
      "logps/rejected": -30.961654663085938,
      "loss": 0.3879,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.3797711730003357,
      "rewards/margins": 0.40343213081359863,
      "rewards/rejected": -0.023660968989133835,
      "step": 630
    },
    {
      "epoch": 1.66,
      "grad_norm": 119.0,
      "learning_rate": 3.6304297682067146e-06,
      "logits/chosen": -2.035832405090332,
      "logits/rejected": -2.032696485519409,
      "logps/chosen": -31.714279174804688,
      "logps/rejected": -32.49519348144531,
      "loss": 0.3205,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.43572768568992615,
      "rewards/margins": 0.5039243102073669,
      "rewards/rejected": -0.06819657981395721,
      "step": 640
    },
    {
      "epoch": 1.69,
      "grad_norm": 24.375,
      "learning_rate": 3.579601087369492e-06,
      "logits/chosen": -2.0391454696655273,
      "logits/rejected": -2.041524887084961,
      "logps/chosen": -33.29151916503906,
      "logps/rejected": -33.737464904785156,
      "loss": 0.2569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.32596832513809204,
      "rewards/margins": 0.37462925910949707,
      "rewards/rejected": -0.04866095632314682,
      "step": 650
    },
    {
      "epoch": 1.71,
      "grad_norm": 32.5,
      "learning_rate": 3.5282177578265295e-06,
      "logits/chosen": -1.9535996913909912,
      "logits/rejected": -1.9540789127349854,
      "logps/chosen": -33.57703399658203,
      "logps/rejected": -31.611495971679688,
      "loss": 0.2509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33281606435775757,
      "rewards/margins": 0.3366300165653229,
      "rewards/rejected": -0.0038140162359923124,
      "step": 660
    },
    {
      "epoch": 1.74,
      "grad_norm": 46.75,
      "learning_rate": 3.476306177936961e-06,
      "logits/chosen": -1.9932886362075806,
      "logits/rejected": -1.983457326889038,
      "logps/chosen": -33.600196838378906,
      "logps/rejected": -32.383522033691406,
      "loss": 0.2964,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.36097538471221924,
      "rewards/margins": 0.4291335642337799,
      "rewards/rejected": -0.06815823167562485,
      "step": 670
    },
    {
      "epoch": 1.77,
      "grad_norm": 53.25,
      "learning_rate": 3.423893017450324e-06,
      "logits/chosen": -1.8898265361785889,
      "logits/rejected": -1.8866018056869507,
      "logps/chosen": -31.05093002319336,
      "logps/rejected": -34.363487243652344,
      "loss": 0.2359,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.36902502179145813,
      "rewards/margins": 0.4094383120536804,
      "rewards/rejected": -0.0404132641851902,
      "step": 680
    },
    {
      "epoch": 1.79,
      "grad_norm": 99.5,
      "learning_rate": 3.3710052038048794e-06,
      "logits/chosen": -1.9469459056854248,
      "logits/rejected": -1.9461286067962646,
      "logps/chosen": -34.85956954956055,
      "logps/rejected": -35.397823333740234,
      "loss": 0.313,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.36964648962020874,
      "rewards/margins": 0.4092490077018738,
      "rewards/rejected": -0.03960253670811653,
      "step": 690
    },
    {
      "epoch": 1.82,
      "grad_norm": 60.0,
      "learning_rate": 3.3176699082935546e-06,
      "logits/chosen": -1.9282480478286743,
      "logits/rejected": -1.9313417673110962,
      "logps/chosen": -32.19511032104492,
      "logps/rejected": -35.74637985229492,
      "loss": 0.2695,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.3669659197330475,
      "rewards/margins": 0.46120119094848633,
      "rewards/rejected": -0.09423529356718063,
      "step": 700
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.247775077819824,
      "eval_logits/rejected": -2.2429134845733643,
      "eval_logps/chosen": -33.943904876708984,
      "eval_logps/rejected": -37.41926193237305,
      "eval_loss": 0.402832955121994,
      "eval_rewards/accuracies": 0.5078904032707214,
      "eval_rewards/chosen": 0.08158490061759949,
      "eval_rewards/margins": -0.006037927698343992,
      "eval_rewards/rejected": 0.08762282878160477,
      "eval_runtime": 145.2777,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.296,
      "step": 700
    },
    {
      "epoch": 1.84,
      "grad_norm": 51.75,
      "learning_rate": 3.2639145321045933e-06,
      "logits/chosen": -2.040821075439453,
      "logits/rejected": -2.0436055660247803,
      "logps/chosen": -34.49801254272461,
      "logps/rejected": -34.32398223876953,
      "loss": 0.3608,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.35781916975975037,
      "rewards/margins": 0.40626224875450134,
      "rewards/rejected": -0.04844306781888008,
      "step": 710
    },
    {
      "epoch": 1.87,
      "grad_norm": 110.5,
      "learning_rate": 3.2097666922441107e-06,
      "logits/chosen": -1.8965297937393188,
      "logits/rejected": -1.8905613422393799,
      "logps/chosen": -34.4133415222168,
      "logps/rejected": -32.38819885253906,
      "loss": 0.2806,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.3779004216194153,
      "rewards/margins": 0.4106220602989197,
      "rewards/rejected": -0.03272164613008499,
      "step": 720
    },
    {
      "epoch": 1.9,
      "grad_norm": 39.5,
      "learning_rate": 3.1552542073477554e-06,
      "logits/chosen": -2.0699164867401123,
      "logits/rejected": -2.0668702125549316,
      "logps/chosen": -30.270557403564453,
      "logps/rejected": -31.52914047241211,
      "loss": 0.2811,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.3441884219646454,
      "rewards/margins": 0.41703981161117554,
      "rewards/rejected": -0.07285144180059433,
      "step": 730
    },
    {
      "epoch": 1.92,
      "grad_norm": 63.0,
      "learning_rate": 3.100405083388799e-06,
      "logits/chosen": -1.9103057384490967,
      "logits/rejected": -1.910112738609314,
      "logps/chosen": -33.07831573486328,
      "logps/rejected": -37.679222106933594,
      "loss": 0.2643,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3463733494281769,
      "rewards/margins": 0.39264675974845886,
      "rewards/rejected": -0.04627341777086258,
      "step": 740
    },
    {
      "epoch": 1.95,
      "grad_norm": 60.75,
      "learning_rate": 3.0452474992899645e-06,
      "logits/chosen": -1.7918742895126343,
      "logits/rejected": -1.7970874309539795,
      "logps/chosen": -36.49778366088867,
      "logps/rejected": -34.00720977783203,
      "loss": 0.2697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3912108838558197,
      "rewards/margins": 0.4225967526435852,
      "rewards/rejected": -0.031385865062475204,
      "step": 750
    },
    {
      "epoch": 1.97,
      "grad_norm": 51.25,
      "learning_rate": 2.989809792446417e-06,
      "logits/chosen": -2.0016860961914062,
      "logits/rejected": -2.003040313720703,
      "logps/chosen": -32.25410461425781,
      "logps/rejected": -32.861629486083984,
      "loss": 0.337,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.31641632318496704,
      "rewards/margins": 0.3905896544456482,
      "rewards/rejected": -0.07417333871126175,
      "step": 760
    },
    {
      "epoch": 2.0,
      "grad_norm": 67.0,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.9714769124984741,
      "logits/rejected": -1.9706627130508423,
      "logps/chosen": -31.806903839111328,
      "logps/rejected": -34.992042541503906,
      "loss": 0.3126,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.3318381905555725,
      "rewards/margins": 0.29933661222457886,
      "rewards/rejected": 0.03250158578157425,
      "step": 770
    },
    {
      "epoch": 2.03,
      "grad_norm": 43.25,
      "learning_rate": 2.878208065043501e-06,
      "logits/chosen": -1.971329689025879,
      "logits/rejected": -1.9705543518066406,
      "logps/chosen": -34.23518371582031,
      "logps/rejected": -31.475284576416016,
      "loss": 0.2613,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.371949702501297,
      "rewards/margins": 0.42213401198387146,
      "rewards/rejected": -0.05018440634012222,
      "step": 780
    },
    {
      "epoch": 2.05,
      "grad_norm": 41.0,
      "learning_rate": 2.8221013802485974e-06,
      "logits/chosen": -2.003086566925049,
      "logits/rejected": -2.0015482902526855,
      "logps/chosen": -29.147836685180664,
      "logps/rejected": -32.72181701660156,
      "loss": 0.1641,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.37381279468536377,
      "rewards/margins": 0.450767457485199,
      "rewards/rejected": -0.07695472240447998,
      "step": 790
    },
    {
      "epoch": 2.08,
      "grad_norm": 28.375,
      "learning_rate": 2.76582921478147e-06,
      "logits/chosen": -2.0545146465301514,
      "logits/rejected": -2.0508596897125244,
      "logps/chosen": -31.972219467163086,
      "logps/rejected": -34.25817108154297,
      "loss": 0.1725,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4094924330711365,
      "rewards/margins": 0.46271318197250366,
      "rewards/rejected": -0.05322074890136719,
      "step": 800
    },
    {
      "epoch": 2.08,
      "eval_logits/chosen": -2.246337890625,
      "eval_logits/rejected": -2.2414615154266357,
      "eval_logps/chosen": -33.92709732055664,
      "eval_logps/rejected": -37.39698791503906,
      "eval_loss": 0.4082644581794739,
      "eval_rewards/accuracies": 0.4821428656578064,
      "eval_rewards/chosen": 0.09670902043581009,
      "eval_rewards/margins": -0.010959767736494541,
      "eval_rewards/rejected": 0.10766878724098206,
      "eval_runtime": 145.3536,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 800
    },
    {
      "epoch": 2.1,
      "grad_norm": 25.625,
      "learning_rate": 2.7094204786572254e-06,
      "logits/chosen": -1.8916804790496826,
      "logits/rejected": -1.8837049007415771,
      "logps/chosen": -34.002410888671875,
      "logps/rejected": -34.613807678222656,
      "loss": 0.2292,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.4084460735321045,
      "rewards/margins": 0.4567646086215973,
      "rewards/rejected": -0.04831857234239578,
      "step": 810
    },
    {
      "epoch": 2.13,
      "grad_norm": 42.0,
      "learning_rate": 2.6529041520546072e-06,
      "logits/chosen": -1.967140555381775,
      "logits/rejected": -1.9777908325195312,
      "logps/chosen": -35.31997299194336,
      "logps/rejected": -32.446533203125,
      "loss": 0.358,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.33814600110054016,
      "rewards/margins": 0.35173219442367554,
      "rewards/rejected": -0.013586151413619518,
      "step": 820
    },
    {
      "epoch": 2.16,
      "grad_norm": 52.0,
      "learning_rate": 2.5963092704273302e-06,
      "logits/chosen": -2.01682710647583,
      "logits/rejected": -2.0210189819335938,
      "logps/chosen": -34.396095275878906,
      "logps/rejected": -29.328195571899414,
      "loss": 0.1479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4560183584690094,
      "rewards/margins": 0.5099755525588989,
      "rewards/rejected": -0.05395715311169624,
      "step": 830
    },
    {
      "epoch": 2.18,
      "grad_norm": 41.25,
      "learning_rate": 2.53966490958702e-06,
      "logits/chosen": -2.016162157058716,
      "logits/rejected": -2.0238842964172363,
      "logps/chosen": -34.38965606689453,
      "logps/rejected": -29.9200439453125,
      "loss": 0.2734,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3585363030433655,
      "rewards/margins": 0.42364779114723206,
      "rewards/rejected": -0.06511145085096359,
      "step": 840
    },
    {
      "epoch": 2.21,
      "grad_norm": 51.0,
      "learning_rate": 2.4830001707654135e-06,
      "logits/chosen": -1.9484409093856812,
      "logits/rejected": -1.9388576745986938,
      "logps/chosen": -31.393484115600586,
      "logps/rejected": -32.06694030761719,
      "loss": 0.1541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.35973984003067017,
      "rewards/margins": 0.40756136178970337,
      "rewards/rejected": -0.04782148450613022,
      "step": 850
    },
    {
      "epoch": 2.23,
      "grad_norm": 28.75,
      "learning_rate": 2.4263441656635054e-06,
      "logits/chosen": -2.0893542766571045,
      "logits/rejected": -2.079577922821045,
      "logps/chosen": -25.801733016967773,
      "logps/rejected": -29.940242767333984,
      "loss": 0.1453,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.38010627031326294,
      "rewards/margins": 0.44122153520584106,
      "rewards/rejected": -0.06111522391438484,
      "step": 860
    },
    {
      "epoch": 2.26,
      "grad_norm": 63.5,
      "learning_rate": 2.3697260014953107e-06,
      "logits/chosen": -1.9405052661895752,
      "logits/rejected": -1.9417035579681396,
      "logps/chosen": -33.93370819091797,
      "logps/rejected": -30.114543914794922,
      "loss": 0.2379,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.35936418175697327,
      "rewards/margins": 0.478200763463974,
      "rewards/rejected": -0.11883661895990372,
      "step": 870
    },
    {
      "epoch": 2.29,
      "grad_norm": 29.25,
      "learning_rate": 2.3131747660339396e-06,
      "logits/chosen": -1.9570789337158203,
      "logits/rejected": -1.957831621170044,
      "logps/chosen": -31.962173461914062,
      "logps/rejected": -33.07716751098633,
      "loss": 0.2589,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3843681216239929,
      "rewards/margins": 0.40799227356910706,
      "rewards/rejected": -0.023624107241630554,
      "step": 880
    },
    {
      "epoch": 2.31,
      "grad_norm": 30.625,
      "learning_rate": 2.256719512667651e-06,
      "logits/chosen": -1.860047698020935,
      "logits/rejected": -1.858214020729065,
      "logps/chosen": -35.269798278808594,
      "logps/rejected": -35.90485382080078,
      "loss": 0.2443,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.3556040823459625,
      "rewards/margins": 0.3804590404033661,
      "rewards/rejected": -0.024854950606822968,
      "step": 890
    },
    {
      "epoch": 2.34,
      "grad_norm": 104.5,
      "learning_rate": 2.2003892454735786e-06,
      "logits/chosen": -2.0041489601135254,
      "logits/rejected": -1.9973175525665283,
      "logps/chosen": -31.497005462646484,
      "logps/rejected": -32.764808654785156,
      "loss": 0.2502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.45506516098976135,
      "rewards/margins": 0.46113377809524536,
      "rewards/rejected": -0.006068611983209848,
      "step": 900
    },
    {
      "epoch": 2.34,
      "eval_logits/chosen": -2.248673439025879,
      "eval_logits/rejected": -2.2437942028045654,
      "eval_logps/chosen": -33.90636444091797,
      "eval_logps/rejected": -37.37092971801758,
      "eval_loss": 0.40991199016571045,
      "eval_rewards/accuracies": 0.49003320932388306,
      "eval_rewards/chosen": 0.11537089198827744,
      "eval_rewards/margins": -0.015749912708997726,
      "eval_rewards/rejected": 0.13112080097198486,
      "eval_runtime": 145.395,
      "eval_samples_per_second": 2.359,
      "eval_steps_per_second": 0.296,
      "step": 900
    },
    {
      "epoch": 2.36,
      "grad_norm": 15.8125,
      "learning_rate": 2.1442129043167877e-06,
      "logits/chosen": -2.0507731437683105,
      "logits/rejected": -2.0459985733032227,
      "logps/chosen": -33.123680114746094,
      "logps/rejected": -34.560203552246094,
      "loss": 0.1461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39060264825820923,
      "rewards/margins": 0.4190269410610199,
      "rewards/rejected": -0.028424326330423355,
      "step": 910
    },
    {
      "epoch": 2.39,
      "grad_norm": 57.25,
      "learning_rate": 2.088219349982323e-06,
      "logits/chosen": -1.9670826196670532,
      "logits/rejected": -1.9720354080200195,
      "logps/chosen": -34.90446090698242,
      "logps/rejected": -32.519283294677734,
      "loss": 0.2881,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3997105062007904,
      "rewards/margins": 0.4171512722969055,
      "rewards/rejected": -0.01744077168405056,
      "step": 920
    },
    {
      "epoch": 2.42,
      "grad_norm": 26.375,
      "learning_rate": 2.0324373493478803e-06,
      "logits/chosen": -2.0626909732818604,
      "logits/rejected": -2.053279161453247,
      "logps/chosen": -31.813243865966797,
      "logps/rejected": -34.46527099609375,
      "loss": 0.1157,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4245172142982483,
      "rewards/margins": 0.44474273920059204,
      "rewards/rejected": -0.020225506275892258,
      "step": 930
    },
    {
      "epoch": 2.44,
      "grad_norm": 85.0,
      "learning_rate": 1.976895560604729e-06,
      "logits/chosen": -1.98918879032135,
      "logits/rejected": -1.986096739768982,
      "logps/chosen": -30.814416885375977,
      "logps/rejected": -32.54015350341797,
      "loss": 0.2202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4266343116760254,
      "rewards/margins": 0.49393558502197266,
      "rewards/rejected": -0.06730121374130249,
      "step": 940
    },
    {
      "epoch": 2.47,
      "grad_norm": 15.1875,
      "learning_rate": 1.921622518534466e-06,
      "logits/chosen": -1.932186484336853,
      "logits/rejected": -1.9396946430206299,
      "logps/chosen": -32.45960235595703,
      "logps/rejected": -35.62566375732422,
      "loss": 0.1934,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3703078627586365,
      "rewards/margins": 0.43846622109413147,
      "rewards/rejected": -0.06815837323665619,
      "step": 950
    },
    {
      "epoch": 2.49,
      "grad_norm": 73.5,
      "learning_rate": 1.8666466198491794e-06,
      "logits/chosen": -1.940987229347229,
      "logits/rejected": -1.934152603149414,
      "logps/chosen": -33.384063720703125,
      "logps/rejected": -36.64484405517578,
      "loss": 0.1978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4000038206577301,
      "rewards/margins": 0.44143620133399963,
      "rewards/rejected": -0.041432395577430725,
      "step": 960
    },
    {
      "epoch": 2.52,
      "grad_norm": 18.0,
      "learning_rate": 1.8119961086025376e-06,
      "logits/chosen": -1.942278265953064,
      "logits/rejected": -1.9429972171783447,
      "logps/chosen": -30.803075790405273,
      "logps/rejected": -33.071388244628906,
      "loss": 0.2326,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.38999220728874207,
      "rewards/margins": 0.47110748291015625,
      "rewards/rejected": -0.0811152458190918,
      "step": 970
    },
    {
      "epoch": 2.55,
      "grad_norm": 31.75,
      "learning_rate": 1.7576990616793139e-06,
      "logits/chosen": -1.946674108505249,
      "logits/rejected": -1.9579198360443115,
      "logps/chosen": -32.60912322998047,
      "logps/rejected": -34.126827239990234,
      "loss": 0.264,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.4208148419857025,
      "rewards/margins": 0.4168703556060791,
      "rewards/rejected": 0.003944486379623413,
      "step": 980
    },
    {
      "epoch": 2.57,
      "grad_norm": 42.0,
      "learning_rate": 1.7037833743707892e-06,
      "logits/chosen": -2.0197396278381348,
      "logits/rejected": -2.017103672027588,
      "logps/chosen": -35.6496696472168,
      "logps/rejected": -32.41356658935547,
      "loss": 0.2859,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.4242428243160248,
      "rewards/margins": 0.4371464252471924,
      "rewards/rejected": -0.012903605587780476,
      "step": 990
    },
    {
      "epoch": 2.6,
      "grad_norm": 35.25,
      "learning_rate": 1.6502767460434588e-06,
      "logits/chosen": -1.9874387979507446,
      "logits/rejected": -1.9911830425262451,
      "logps/chosen": -34.0231819152832,
      "logps/rejected": -34.56896209716797,
      "loss": 0.1529,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.39610037207603455,
      "rewards/margins": 0.47072356939315796,
      "rewards/rejected": -0.07462319731712341,
      "step": 1000
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.2477176189422607,
      "eval_logits/rejected": -2.242847442626953,
      "eval_logps/chosen": -33.89875411987305,
      "eval_logps/rejected": -37.377010345458984,
      "eval_loss": 0.3879217505455017,
      "eval_rewards/accuracies": 0.5215947031974792,
      "eval_rewards/chosen": 0.12221997231245041,
      "eval_rewards/margins": -0.003430851735174656,
      "eval_rewards/rejected": 0.1256508231163025,
      "eval_runtime": 145.3271,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 1000
    },
    {
      "epoch": 2.62,
      "grad_norm": 40.25,
      "learning_rate": 1.5972066659083796e-06,
      "logits/chosen": -1.9966695308685303,
      "logits/rejected": -2.0017590522766113,
      "logps/chosen": -32.33439636230469,
      "logps/rejected": -32.539424896240234,
      "loss": 0.1576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41942542791366577,
      "rewards/margins": 0.43577662110328674,
      "rewards/rejected": -0.01635116897523403,
      "step": 1010
    },
    {
      "epoch": 2.65,
      "grad_norm": 53.25,
      "learning_rate": 1.5446003988985041e-06,
      "logits/chosen": -1.959848165512085,
      "logits/rejected": -1.962561011314392,
      "logps/chosen": -30.024948120117188,
      "logps/rejected": -31.70804786682129,
      "loss": 0.2273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3786233961582184,
      "rewards/margins": 0.3866207003593445,
      "rewards/rejected": -0.007997283712029457,
      "step": 1020
    },
    {
      "epoch": 2.68,
      "grad_norm": 23.125,
      "learning_rate": 1.4924849716612211e-06,
      "logits/chosen": -2.017338514328003,
      "logits/rejected": -2.0114893913269043,
      "logps/chosen": -33.496673583984375,
      "logps/rejected": -33.55792999267578,
      "loss": 0.1431,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4431969225406647,
      "rewards/margins": 0.4791356027126312,
      "rewards/rejected": -0.035938672721385956,
      "step": 1030
    },
    {
      "epoch": 2.7,
      "grad_norm": 56.0,
      "learning_rate": 1.440887158673332e-06,
      "logits/chosen": -1.9453651905059814,
      "logits/rejected": -1.9490783214569092,
      "logps/chosen": -35.259788513183594,
      "logps/rejected": -34.503028869628906,
      "loss": 0.2102,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4445478916168213,
      "rewards/margins": 0.519649863243103,
      "rewards/rejected": -0.07510194182395935,
      "step": 1040
    },
    {
      "epoch": 2.73,
      "grad_norm": 35.75,
      "learning_rate": 1.3898334684855647e-06,
      "logits/chosen": -1.9455549716949463,
      "logits/rejected": -1.95868718624115,
      "logps/chosen": -31.420841217041016,
      "logps/rejected": -33.076026916503906,
      "loss": 0.2334,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.39784207940101624,
      "rewards/margins": 0.4482339024543762,
      "rewards/rejected": -0.05039183050394058,
      "step": 1050
    },
    {
      "epoch": 2.75,
      "grad_norm": 58.75,
      "learning_rate": 1.3393501301037245e-06,
      "logits/chosen": -1.9375944137573242,
      "logits/rejected": -1.9310089349746704,
      "logps/chosen": -31.681589126586914,
      "logps/rejected": -33.36168670654297,
      "loss": 0.1253,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.42997032403945923,
      "rewards/margins": 0.46931910514831543,
      "rewards/rejected": -0.03934880718588829,
      "step": 1060
    },
    {
      "epoch": 2.78,
      "grad_norm": 40.5,
      "learning_rate": 1.2894630795134454e-06,
      "logits/chosen": -2.0641446113586426,
      "logits/rejected": -2.0649874210357666,
      "logps/chosen": -33.09912109375,
      "logps/rejected": -32.90647888183594,
      "loss": 0.1818,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.4031626284122467,
      "rewards/margins": 0.4587678909301758,
      "rewards/rejected": -0.05560523271560669,
      "step": 1070
    },
    {
      "epoch": 2.81,
      "grad_norm": 25.25,
      "learning_rate": 1.2401979463554984e-06,
      "logits/chosen": -1.9966720342636108,
      "logits/rejected": -1.9955018758773804,
      "logps/chosen": -33.611534118652344,
      "logps/rejected": -33.192691802978516,
      "loss": 0.1975,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.43046802282333374,
      "rewards/margins": 0.431241899728775,
      "rewards/rejected": -0.0007738359272480011,
      "step": 1080
    },
    {
      "epoch": 2.83,
      "grad_norm": 24.625,
      "learning_rate": 1.1915800407584705e-06,
      "logits/chosen": -2.0245463848114014,
      "logits/rejected": -2.0170249938964844,
      "logps/chosen": -33.81514358520508,
      "logps/rejected": -31.061843872070312,
      "loss": 0.1443,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.3696584105491638,
      "rewards/margins": 0.387938916683197,
      "rewards/rejected": -0.018280521035194397,
      "step": 1090
    },
    {
      "epoch": 2.86,
      "grad_norm": 88.0,
      "learning_rate": 1.1436343403356019e-06,
      "logits/chosen": -1.9909054040908813,
      "logits/rejected": -1.9901247024536133,
      "logps/chosen": -34.939388275146484,
      "logps/rejected": -36.37110137939453,
      "loss": 0.1583,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.43112340569496155,
      "rewards/margins": 0.44767728447914124,
      "rewards/rejected": -0.01655389368534088,
      "step": 1100
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.2481679916381836,
      "eval_logits/rejected": -2.243297576904297,
      "eval_logps/chosen": -33.90199279785156,
      "eval_logps/rejected": -37.37769317626953,
      "eval_loss": 0.3968319892883301,
      "eval_rewards/accuracies": 0.4875415563583374,
      "eval_rewards/chosen": 0.11930261552333832,
      "eval_rewards/margins": -0.0057318080216646194,
      "eval_rewards/rejected": 0.1250344216823578,
      "eval_runtime": 145.2343,
      "eval_samples_per_second": 2.362,
      "eval_steps_per_second": 0.296,
      "step": 1100
    },
    {
      "epoch": 2.88,
      "grad_norm": 34.0,
      "learning_rate": 1.0963854773524548e-06,
      "logits/chosen": -2.0373494625091553,
      "logits/rejected": -2.0427050590515137,
      "logps/chosen": -35.03519821166992,
      "logps/rejected": -35.56550216674805,
      "loss": 0.1969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38872379064559937,
      "rewards/margins": 0.40845227241516113,
      "rewards/rejected": -0.01972847804427147,
      "step": 1110
    },
    {
      "epoch": 2.91,
      "grad_norm": 91.0,
      "learning_rate": 1.049857726072005e-06,
      "logits/chosen": -2.009969472885132,
      "logits/rejected": -2.0080225467681885,
      "logps/chosen": -32.2147216796875,
      "logps/rejected": -32.992332458496094,
      "loss": 0.2049,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.36133700609207153,
      "rewards/margins": 0.36995649337768555,
      "rewards/rejected": -0.008619492873549461,
      "step": 1120
    },
    {
      "epoch": 2.94,
      "grad_norm": 42.0,
      "learning_rate": 1.0040749902836508e-06,
      "logits/chosen": -1.9126440286636353,
      "logits/rejected": -1.9155027866363525,
      "logps/chosen": -29.281444549560547,
      "logps/rejected": -30.237567901611328,
      "loss": 0.18,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.42047175765037537,
      "rewards/margins": 0.4137454926967621,
      "rewards/rejected": 0.006726269610226154,
      "step": 1130
    },
    {
      "epoch": 2.96,
      "grad_norm": 33.0,
      "learning_rate": 9.59060791022566e-07,
      "logits/chosen": -2.009428024291992,
      "logits/rejected": -2.0063159465789795,
      "logps/chosen": -32.74279022216797,
      "logps/rejected": -32.62069320678711,
      "loss": 0.1567,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.4148639142513275,
      "rewards/margins": 0.49929580092430115,
      "rewards/rejected": -0.08443190902471542,
      "step": 1140
    },
    {
      "epoch": 2.99,
      "grad_norm": 50.25,
      "learning_rate": 9.148382544856885e-07,
      "logits/chosen": -1.931823968887329,
      "logits/rejected": -1.925490140914917,
      "logps/chosen": -27.730154037475586,
      "logps/rejected": -32.26335906982422,
      "loss": 0.1296,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3670499324798584,
      "rewards/margins": 0.47060221433639526,
      "rewards/rejected": -0.10355228185653687,
      "step": 1150
    },
    {
      "epoch": 3.01,
      "grad_norm": 35.25,
      "learning_rate": 8.714301001505568e-07,
      "logits/chosen": -2.0501132011413574,
      "logits/rejected": -2.047624111175537,
      "logps/chosen": -32.637855529785156,
      "logps/rejected": -35.0833625793457,
      "loss": 0.0915,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.41239070892333984,
      "rewards/margins": 0.4050343632698059,
      "rewards/rejected": 0.007356312125921249,
      "step": 1160
    },
    {
      "epoch": 3.04,
      "grad_norm": 17.625,
      "learning_rate": 8.288586291031025e-07,
      "logits/chosen": -2.0300636291503906,
      "logits/rejected": -2.0281882286071777,
      "logps/chosen": -31.161453247070312,
      "logps/rejected": -32.45496368408203,
      "loss": 0.0941,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4076564908027649,
      "rewards/margins": 0.4894997179508209,
      "rewards/rejected": -0.08184324204921722,
      "step": 1170
    },
    {
      "epoch": 3.06,
      "grad_norm": 32.25,
      "learning_rate": 7.871457125803897e-07,
      "logits/chosen": -2.02311110496521,
      "logits/rejected": -2.0107522010803223,
      "logps/chosen": -35.538490295410156,
      "logps/rejected": -34.12282180786133,
      "loss": 0.1417,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4561285972595215,
      "rewards/margins": 0.5023427605628967,
      "rewards/rejected": -0.046214111149311066,
      "step": 1180
    },
    {
      "epoch": 3.09,
      "grad_norm": 55.0,
      "learning_rate": 7.463127807341966e-07,
      "logits/chosen": -1.9117752313613892,
      "logits/rejected": -1.9064388275146484,
      "logps/chosen": -34.895999908447266,
      "logps/rejected": -34.239891052246094,
      "loss": 0.1368,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4083231985569,
      "rewards/margins": 0.41019004583358765,
      "rewards/rejected": -0.001866865553893149,
      "step": 1190
    },
    {
      "epoch": 3.12,
      "grad_norm": 54.5,
      "learning_rate": 7.063808116212021e-07,
      "logits/chosen": -1.9550445079803467,
      "logits/rejected": -1.958034873008728,
      "logps/chosen": -32.1730842590332,
      "logps/rejected": -31.49075698852539,
      "loss": 0.113,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.39780890941619873,
      "rewards/margins": 0.4019277095794678,
      "rewards/rejected": -0.004118794109672308,
      "step": 1200
    },
    {
      "epoch": 3.12,
      "eval_logits/chosen": -2.2470004558563232,
      "eval_logits/rejected": -2.2421324253082275,
      "eval_logps/chosen": -33.908164978027344,
      "eval_logps/rejected": -37.38742446899414,
      "eval_loss": 0.3848971724510193,
      "eval_rewards/accuracies": 0.47840529680252075,
      "eval_rewards/chosen": 0.11374907940626144,
      "eval_rewards/margins": -0.002530111465603113,
      "eval_rewards/rejected": 0.11627919226884842,
      "eval_runtime": 145.1536,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1200
    },
    {
      "epoch": 3.14,
      "grad_norm": 23.875,
      "learning_rate": 6.673703204254348e-07,
      "logits/chosen": -1.9873783588409424,
      "logits/rejected": -1.9827861785888672,
      "logps/chosen": -30.8137264251709,
      "logps/rejected": -30.97092628479004,
      "loss": 0.1001,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.41394877433776855,
      "rewards/margins": 0.4704950451850891,
      "rewards/rejected": -0.05654625967144966,
      "step": 1210
    },
    {
      "epoch": 3.17,
      "grad_norm": 18.875,
      "learning_rate": 6.293013489185315e-07,
      "logits/chosen": -1.9986066818237305,
      "logits/rejected": -1.9930963516235352,
      "logps/chosen": -34.82990264892578,
      "logps/rejected": -34.601905822753906,
      "loss": 0.1136,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.399636447429657,
      "rewards/margins": 0.4284568727016449,
      "rewards/rejected": -0.02882050909101963,
      "step": 1220
    },
    {
      "epoch": 3.19,
      "grad_norm": 35.0,
      "learning_rate": 5.921934551632086e-07,
      "logits/chosen": -2.000286102294922,
      "logits/rejected": -1.9870882034301758,
      "logps/chosen": -32.6419677734375,
      "logps/rejected": -34.18825149536133,
      "loss": 0.1139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4068435728549957,
      "rewards/margins": 0.4684053957462311,
      "rewards/rejected": -0.06156182289123535,
      "step": 1230
    },
    {
      "epoch": 3.22,
      "grad_norm": 29.0,
      "learning_rate": 5.560657034652405e-07,
      "logits/chosen": -2.044322967529297,
      "logits/rejected": -2.0414676666259766,
      "logps/chosen": -34.62710189819336,
      "logps/rejected": -31.878192901611328,
      "loss": 0.1302,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.36521658301353455,
      "rewards/margins": 0.399711012840271,
      "rewards/rejected": -0.03449442610144615,
      "step": 1240
    },
    {
      "epoch": 3.25,
      "grad_norm": 51.5,
      "learning_rate": 5.2093665457911e-07,
      "logits/chosen": -1.9766407012939453,
      "logits/rejected": -1.9737846851348877,
      "logps/chosen": -33.98097610473633,
      "logps/rejected": -34.84381103515625,
      "loss": 0.1699,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.37366223335266113,
      "rewards/margins": 0.4156786799430847,
      "rewards/rejected": -0.04201648384332657,
      "step": 1250
    },
    {
      "epoch": 3.27,
      "grad_norm": 30.75,
      "learning_rate": 4.868243561723535e-07,
      "logits/chosen": -2.038552761077881,
      "logits/rejected": -2.0333991050720215,
      "logps/chosen": -30.33209800720215,
      "logps/rejected": -32.44548797607422,
      "loss": 0.0978,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3858305513858795,
      "rewards/margins": 0.44247692823410034,
      "rewards/rejected": -0.05664638429880142,
      "step": 1260
    },
    {
      "epoch": 3.3,
      "grad_norm": 28.375,
      "learning_rate": 4.537463335535161e-07,
      "logits/chosen": -2.065164089202881,
      "logits/rejected": -2.069824457168579,
      "logps/chosen": -32.27547073364258,
      "logps/rejected": -31.700992584228516,
      "loss": 0.1001,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3919737637042999,
      "rewards/margins": 0.4096389412879944,
      "rewards/rejected": -0.017665188759565353,
      "step": 1270
    },
    {
      "epoch": 3.32,
      "grad_norm": 18.625,
      "learning_rate": 4.217195806684629e-07,
      "logits/chosen": -1.9599748849868774,
      "logits/rejected": -1.9670900106430054,
      "logps/chosen": -34.481956481933594,
      "logps/rejected": -32.67145538330078,
      "loss": 0.1083,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.3827614486217499,
      "rewards/margins": 0.4486331343650818,
      "rewards/rejected": -0.06587164849042892,
      "step": 1280
    },
    {
      "epoch": 3.35,
      "grad_norm": 33.25,
      "learning_rate": 3.907605513696808e-07,
      "logits/chosen": -1.862562894821167,
      "logits/rejected": -1.864749550819397,
      "logps/chosen": -32.70376968383789,
      "logps/rejected": -36.58349609375,
      "loss": 0.1038,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3853146433830261,
      "rewards/margins": 0.41162100434303284,
      "rewards/rejected": -0.026306327432394028,
      "step": 1290
    },
    {
      "epoch": 3.38,
      "grad_norm": 64.0,
      "learning_rate": 3.6088515096305675e-07,
      "logits/chosen": -1.9185287952423096,
      "logits/rejected": -1.921884298324585,
      "logps/chosen": -32.8210334777832,
      "logps/rejected": -32.53430938720703,
      "loss": 0.0937,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.35607367753982544,
      "rewards/margins": 0.41920334100723267,
      "rewards/rejected": -0.06312969326972961,
      "step": 1300
    },
    {
      "epoch": 3.38,
      "eval_logits/chosen": -2.2471628189086914,
      "eval_logits/rejected": -2.2422919273376465,
      "eval_logps/chosen": -33.897342681884766,
      "eval_logps/rejected": -37.385860443115234,
      "eval_loss": 0.37376365065574646,
      "eval_rewards/accuracies": 0.5045680999755859,
      "eval_rewards/chosen": 0.1234876811504364,
      "eval_rewards/margins": 0.005806357134133577,
      "eval_rewards/rejected": 0.1176813393831253,
      "eval_runtime": 145.1552,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1300
    },
    {
      "epoch": 3.4,
      "grad_norm": 37.5,
      "learning_rate": 3.321087280364757e-07,
      "logits/chosen": -2.027667999267578,
      "logits/rejected": -2.0075087547302246,
      "logps/chosen": -30.6841983795166,
      "logps/rejected": -35.201026916503906,
      "loss": 0.135,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.38628271222114563,
      "rewards/margins": 0.4174365997314453,
      "rewards/rejected": -0.031153928488492966,
      "step": 1310
    },
    {
      "epoch": 3.43,
      "grad_norm": 141.0,
      "learning_rate": 3.044460665744284e-07,
      "logits/chosen": -2.0063624382019043,
      "logits/rejected": -2.012112855911255,
      "logps/chosen": -32.609859466552734,
      "logps/rejected": -32.543277740478516,
      "loss": 0.1282,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.38153642416000366,
      "rewards/margins": 0.4288099706172943,
      "rewards/rejected": -0.04727357253432274,
      "step": 1320
    },
    {
      "epoch": 3.45,
      "grad_norm": 13.625,
      "learning_rate": 2.779113783626916e-07,
      "logits/chosen": -1.9746071100234985,
      "logits/rejected": -1.969681739807129,
      "logps/chosen": -32.591461181640625,
      "logps/rejected": -34.600608825683594,
      "loss": 0.1059,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.38793960213661194,
      "rewards/margins": 0.4134295880794525,
      "rewards/rejected": -0.025490064173936844,
      "step": 1330
    },
    {
      "epoch": 3.48,
      "grad_norm": 35.75,
      "learning_rate": 2.5251829568697204e-07,
      "logits/chosen": -1.8414928913116455,
      "logits/rejected": -1.8512153625488281,
      "logps/chosen": -33.299381256103516,
      "logps/rejected": -31.048131942749023,
      "loss": 0.0735,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.4501881003379822,
      "rewards/margins": 0.4628051221370697,
      "rewards/rejected": -0.012616944499313831,
      "step": 1340
    },
    {
      "epoch": 3.51,
      "grad_norm": 22.5,
      "learning_rate": 2.2827986432927774e-07,
      "logits/chosen": -1.9015308618545532,
      "logits/rejected": -1.9062535762786865,
      "logps/chosen": -33.4455680847168,
      "logps/rejected": -33.258026123046875,
      "loss": 0.0823,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.4548335671424866,
      "rewards/margins": 0.5027266144752502,
      "rewards/rejected": -0.04789309203624725,
      "step": 1350
    },
    {
      "epoch": 3.53,
      "grad_norm": 56.25,
      "learning_rate": 2.0520853686560177e-07,
      "logits/chosen": -1.9281432628631592,
      "logits/rejected": -1.9337003231048584,
      "logps/chosen": -33.40667724609375,
      "logps/rejected": -34.605411529541016,
      "loss": 0.0912,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.38248568773269653,
      "rewards/margins": 0.42059844732284546,
      "rewards/rejected": -0.038112785667181015,
      "step": 1360
    },
    {
      "epoch": 3.56,
      "grad_norm": 11.375,
      "learning_rate": 1.833161662683672e-07,
      "logits/chosen": -2.017787456512451,
      "logits/rejected": -2.0132956504821777,
      "logps/chosen": -32.1794319152832,
      "logps/rejected": -30.92777442932129,
      "loss": 0.1078,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4106277525424957,
      "rewards/margins": 0.4249129891395569,
      "rewards/rejected": -0.014285224489867687,
      "step": 1370
    },
    {
      "epoch": 3.58,
      "grad_norm": 19.875,
      "learning_rate": 1.626139998169246e-07,
      "logits/chosen": -1.9410808086395264,
      "logits/rejected": -1.9427919387817383,
      "logps/chosen": -29.26833152770996,
      "logps/rejected": -30.779708862304688,
      "loss": 0.0842,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.35187023878097534,
      "rewards/margins": 0.3838461935520172,
      "rewards/rejected": -0.031975992023944855,
      "step": 1380
    },
    {
      "epoch": 3.61,
      "grad_norm": 16.125,
      "learning_rate": 1.4311267331922535e-07,
      "logits/chosen": -1.926817536354065,
      "logits/rejected": -1.931318998336792,
      "logps/chosen": -31.545446395874023,
      "logps/rejected": -33.034786224365234,
      "loss": 0.0861,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.4266428053379059,
      "rewards/margins": 0.46869271993637085,
      "rewards/rejected": -0.042049963027238846,
      "step": 1390
    },
    {
      "epoch": 3.64,
      "grad_norm": 29.0,
      "learning_rate": 1.2482220564763669e-07,
      "logits/chosen": -1.9985454082489014,
      "logits/rejected": -2.00184965133667,
      "logps/chosen": -34.45569610595703,
      "logps/rejected": -33.49158477783203,
      "loss": 0.0815,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4273880422115326,
      "rewards/margins": 0.4886067509651184,
      "rewards/rejected": -0.061218757182359695,
      "step": 1400
    },
    {
      "epoch": 3.64,
      "eval_logits/chosen": -2.2475900650024414,
      "eval_logits/rejected": -2.2427213191986084,
      "eval_logps/chosen": -33.8858757019043,
      "eval_logps/rejected": -37.38360595703125,
      "eval_loss": 0.35954806208610535,
      "eval_rewards/accuracies": 0.5224252939224243,
      "eval_rewards/chosen": 0.13380877673625946,
      "eval_rewards/margins": 0.014098312705755234,
      "eval_rewards/rejected": 0.11971045285463333,
      "eval_runtime": 145.2937,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.296,
      "step": 1400
    },
    {
      "epoch": 3.66,
      "grad_norm": 38.25,
      "learning_rate": 1.0775199359171346e-07,
      "logits/chosen": -2.0676581859588623,
      "logits/rejected": -2.060824155807495,
      "logps/chosen": -33.353126525878906,
      "logps/rejected": -34.27102279663086,
      "loss": 0.1028,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.4044764041900635,
      "rewards/margins": 0.42828792333602905,
      "rewards/rejected": -0.023811563849449158,
      "step": 1410
    },
    {
      "epoch": 3.69,
      "grad_norm": 46.0,
      "learning_rate": 9.191080703056604e-08,
      "logits/chosen": -1.9524263143539429,
      "logits/rejected": -1.9631872177124023,
      "logps/chosen": -33.7514533996582,
      "logps/rejected": -33.68230438232422,
      "loss": 0.1069,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3580663800239563,
      "rewards/margins": 0.44016751646995544,
      "rewards/rejected": -0.08210111409425735,
      "step": 1420
    },
    {
      "epoch": 3.71,
      "grad_norm": 33.0,
      "learning_rate": 7.730678442730539e-08,
      "logits/chosen": -2.038909912109375,
      "logits/rejected": -2.0507607460021973,
      "logps/chosen": -34.61919021606445,
      "logps/rejected": -33.713623046875,
      "loss": 0.1328,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.39745327830314636,
      "rewards/margins": 0.465212881565094,
      "rewards/rejected": -0.06775961816310883,
      "step": 1430
    },
    {
      "epoch": 3.74,
      "grad_norm": 20.125,
      "learning_rate": 6.394742864787806e-08,
      "logits/chosen": -2.0287575721740723,
      "logits/rejected": -2.030729055404663,
      "logps/chosen": -32.74304962158203,
      "logps/rejected": -33.90821075439453,
      "loss": 0.1046,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4188024401664734,
      "rewards/margins": 0.46222352981567383,
      "rewards/rejected": -0.04342114180326462,
      "step": 1440
    },
    {
      "epoch": 3.77,
      "grad_norm": 20.875,
      "learning_rate": 5.183960310644748e-08,
      "logits/chosen": -2.010601758956909,
      "logits/rejected": -2.0033953189849854,
      "logps/chosen": -35.09027862548828,
      "logps/rejected": -33.78215789794922,
      "loss": 0.0945,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.43024691939353943,
      "rewards/margins": 0.4408265948295593,
      "rewards/rejected": -0.010579647496342659,
      "step": 1450
    },
    {
      "epoch": 3.79,
      "grad_norm": 58.5,
      "learning_rate": 4.098952823928693e-08,
      "logits/chosen": -1.9762957096099854,
      "logits/rejected": -1.9823601245880127,
      "logps/chosen": -30.78021240234375,
      "logps/rejected": -33.74589538574219,
      "loss": 0.0957,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.38966792821884155,
      "rewards/margins": 0.45797547698020935,
      "rewards/rejected": -0.06830750405788422,
      "step": 1460
    },
    {
      "epoch": 3.82,
      "grad_norm": 28.375,
      "learning_rate": 3.1402778309014284e-08,
      "logits/chosen": -1.9236361980438232,
      "logits/rejected": -1.9213138818740845,
      "logps/chosen": -30.394033432006836,
      "logps/rejected": -30.590625762939453,
      "loss": 0.1267,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.40636783838272095,
      "rewards/margins": 0.4031190872192383,
      "rewards/rejected": 0.003248719498515129,
      "step": 1470
    },
    {
      "epoch": 3.84,
      "grad_norm": 18.25,
      "learning_rate": 2.3084278540791427e-08,
      "logits/chosen": -2.09725284576416,
      "logits/rejected": -2.091542959213257,
      "logps/chosen": -34.822288513183594,
      "logps/rejected": -32.077293395996094,
      "loss": 0.0885,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4166763424873352,
      "rewards/margins": 0.4352034032344818,
      "rewards/rejected": -0.018527105450630188,
      "step": 1480
    },
    {
      "epoch": 3.87,
      "grad_norm": 16.0,
      "learning_rate": 1.6038302591975807e-08,
      "logits/chosen": -1.9885956048965454,
      "logits/rejected": -1.9906467199325562,
      "logps/chosen": -28.027481079101562,
      "logps/rejected": -28.051410675048828,
      "loss": 0.0847,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.40400925278663635,
      "rewards/margins": 0.4302259087562561,
      "rewards/rejected": -0.02621668018400669,
      "step": 1490
    },
    {
      "epoch": 3.9,
      "grad_norm": 28.375,
      "learning_rate": 1.0268470356514237e-08,
      "logits/chosen": -1.9982271194458008,
      "logits/rejected": -1.9925235509872437,
      "logps/chosen": -32.401649475097656,
      "logps/rejected": -32.05929946899414,
      "loss": 0.0757,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.359231173992157,
      "rewards/margins": 0.44054365158081055,
      "rewards/rejected": -0.08131246268749237,
      "step": 1500
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -2.2469353675842285,
      "eval_logits/rejected": -2.2420578002929688,
      "eval_logps/chosen": -33.8865852355957,
      "eval_logps/rejected": -37.384151458740234,
      "eval_loss": 0.3542863130569458,
      "eval_rewards/accuracies": 0.5485880374908447,
      "eval_rewards/chosen": 0.1331672966480255,
      "eval_rewards/margins": 0.013947191648185253,
      "eval_rewards/rejected": 0.11922012269496918,
      "eval_runtime": 145.2538,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.296,
      "step": 1500
    },
    {
      "epoch": 3.92,
      "grad_norm": 30.375,
      "learning_rate": 5.777746105209147e-09,
      "logits/chosen": -1.9242042303085327,
      "logits/rejected": -1.9285516738891602,
      "logps/chosen": -33.746910095214844,
      "logps/rejected": -34.46141815185547,
      "loss": 0.1179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38167905807495117,
      "rewards/margins": 0.4276120662689209,
      "rewards/rejected": -0.04593303054571152,
      "step": 1510
    },
    {
      "epoch": 3.95,
      "grad_norm": 78.5,
      "learning_rate": 2.5684369628148352e-09,
      "logits/chosen": -1.9852806329727173,
      "logits/rejected": -1.9835399389266968,
      "logps/chosen": -30.321674346923828,
      "logps/rejected": -32.68654251098633,
      "loss": 0.1491,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.3712538182735443,
      "rewards/margins": 0.4583936631679535,
      "rewards/rejected": -0.08713988214731216,
      "step": 1520
    },
    {
      "epoch": 3.97,
      "grad_norm": 41.25,
      "learning_rate": 6.421917227455999e-10,
      "logits/chosen": -2.082426071166992,
      "logits/rejected": -2.074831008911133,
      "logps/chosen": -27.650592803955078,
      "logps/rejected": -28.735992431640625,
      "loss": 0.1085,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3711165487766266,
      "rewards/margins": 0.4290649890899658,
      "rewards/rejected": -0.05794848874211311,
      "step": 1530
    },
    {
      "epoch": 4.0,
      "grad_norm": 50.25,
      "learning_rate": 0.0,
      "logits/chosen": -1.9804470539093018,
      "logits/rejected": -1.9702799320220947,
      "logps/chosen": -32.56533432006836,
      "logps/rejected": -35.504661560058594,
      "loss": 0.1078,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.4507782459259033,
      "rewards/margins": 0.5349205732345581,
      "rewards/rejected": -0.0841422826051712,
      "step": 1540
    },
    {
      "epoch": 4.0,
      "step": 1540,
      "total_flos": 0.0,
      "train_loss": 0.17404492969636792,
      "train_runtime": 10767.1792,
      "train_samples_per_second": 1.144,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 1540,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}