{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.3086,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8665881156921387,
      "logits/rejected": -1.8709055185317993,
      "logps/chosen": -36.99662399291992,
      "logps/rejected": -33.65571594238281,
      "loss": 0.3052,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.008980684913694859,
      "rewards/margins": 0.032059140503406525,
      "rewards/rejected": -0.02307845838367939,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9977455139160156,
      "logits/rejected": -2.000382423400879,
      "logps/chosen": -29.642925262451172,
      "logps/rejected": -29.056737899780273,
      "loss": 0.3525,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0006573178106918931,
      "rewards/margins": -0.00669272243976593,
      "rewards/rejected": 0.006035405211150646,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9204607009887695,
      "logits/rejected": -1.9177772998809814,
      "logps/chosen": -31.42336654663086,
      "logps/rejected": -33.22785568237305,
      "loss": 0.3603,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.006546213291585445,
      "rewards/margins": 0.0019128695130348206,
      "rewards/rejected": -0.00845908559858799,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0172362327575684,
      "logits/rejected": -2.008507251739502,
      "logps/chosen": -32.56964874267578,
      "logps/rejected": -32.50572967529297,
      "loss": 0.3558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006439635064452887,
      "rewards/margins": 0.00019515231542754918,
      "rewards/rejected": 0.006244482938200235,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8618619441986084,
      "logits/rejected": -1.8510783910751343,
      "logps/chosen": -33.56026077270508,
      "logps/rejected": -35.45254898071289,
      "loss": 0.3691,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0025013976264744997,
      "rewards/margins": -0.002315213903784752,
      "rewards/rejected": -0.00018618404283188283,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9393202066421509,
      "logits/rejected": -1.9412600994110107,
      "logps/chosen": -32.57838439941406,
      "logps/rejected": -33.215576171875,
      "loss": 0.3063,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.01962057128548622,
      "rewards/margins": 0.05940054729580879,
      "rewards/rejected": -0.03977997973561287,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.0718436241149902,
      "logits/rejected": -2.0768017768859863,
      "logps/chosen": -33.97806167602539,
      "logps/rejected": -36.63082504272461,
      "loss": 0.4257,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00938049890100956,
      "rewards/margins": 0.06513925641775131,
      "rewards/rejected": -0.0557587556540966,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9351739883422852,
      "logits/rejected": -1.9383188486099243,
      "logps/chosen": -34.33073043823242,
      "logps/rejected": -34.61904525756836,
      "loss": 0.2902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08736880123615265,
      "rewards/margins": 0.11134655773639679,
      "rewards/rejected": -0.023977745324373245,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.944392204284668,
      "logits/rejected": -1.9489190578460693,
      "logps/chosen": -32.419586181640625,
      "logps/rejected": -32.3698844909668,
      "loss": 0.3611,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.043051257729530334,
      "rewards/margins": 0.052930813282728195,
      "rewards/rejected": -0.009879561141133308,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.0419769287109375,
      "logits/rejected": -2.0399627685546875,
      "logps/chosen": -32.174407958984375,
      "logps/rejected": -31.26608657836914,
      "loss": 0.3038,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.059785228222608566,
      "rewards/margins": 0.05967242643237114,
      "rewards/rejected": 0.00011279433965682983,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2372143268585205,
      "eval_logits/rejected": -2.2323503494262695,
      "eval_logps/chosen": -34.01276779174805,
      "eval_logps/rejected": -37.51152420043945,
      "eval_loss": 0.3511974811553955,
      "eval_rewards/accuracies": 0.5423588156700134,
      "eval_rewards/chosen": 0.01960929110646248,
      "eval_rewards/margins": 0.01502405758947134,
      "eval_rewards/rejected": 0.004585230257362127,
      "eval_runtime": 145.9032,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.998891830444336,
      "logits/rejected": -1.996492624282837,
      "logps/chosen": -33.14598083496094,
      "logps/rejected": -34.020729064941406,
      "loss": 0.4689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.08805312216281891,
      "rewards/margins": 0.08027410507202148,
      "rewards/rejected": 0.007779018487781286,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.010932445526123,
      "logits/rejected": -2.0025696754455566,
      "logps/chosen": -32.37172317504883,
      "logps/rejected": -32.118797302246094,
      "loss": 0.4465,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.06586603820323944,
      "rewards/margins": 0.02742874063551426,
      "rewards/rejected": 0.03843729570508003,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0387539863586426,
      "logits/rejected": -2.030724287033081,
      "logps/chosen": -30.41655921936035,
      "logps/rejected": -32.060333251953125,
      "loss": 0.3844,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.05324209854006767,
      "rewards/margins": 0.05983499437570572,
      "rewards/rejected": -0.00659290561452508,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.967858076095581,
      "logits/rejected": -1.9781148433685303,
      "logps/chosen": -31.223413467407227,
      "logps/rejected": -32.55517578125,
      "loss": 0.3904,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1766217201948166,
      "rewards/margins": 0.1948881596326828,
      "rewards/rejected": -0.01826643943786621,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.880910873413086,
      "logits/rejected": -1.8820507526397705,
      "logps/chosen": -34.01464080810547,
      "logps/rejected": -34.783546447753906,
      "loss": 0.3871,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.160243421792984,
      "rewards/margins": 0.1839032918214798,
      "rewards/rejected": -0.023659853264689445,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.933895468711853,
      "logits/rejected": -1.9304730892181396,
      "logps/chosen": -36.02853775024414,
      "logps/rejected": -32.699058532714844,
      "loss": 0.2729,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1288261115550995,
      "rewards/margins": 0.10750452429056168,
      "rewards/rejected": 0.021321602165699005,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0341715812683105,
      "logits/rejected": -2.0267820358276367,
      "logps/chosen": -33.55347442626953,
      "logps/rejected": -31.3526554107666,
      "loss": 0.294,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.21994857490062714,
      "rewards/margins": 0.22067300975322723,
      "rewards/rejected": -0.0007244400912895799,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0400891304016113,
      "logits/rejected": -2.045360565185547,
      "logps/chosen": -32.370338439941406,
      "logps/rejected": -32.4719123840332,
      "loss": 0.2829,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.15700635313987732,
      "rewards/margins": 0.145101398229599,
      "rewards/rejected": 0.011904975399374962,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.041393280029297,
      "logits/rejected": -2.038623809814453,
      "logps/chosen": -31.328174591064453,
      "logps/rejected": -31.316492080688477,
      "loss": 0.3044,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.14470075070858002,
      "rewards/margins": 0.1431477963924408,
      "rewards/rejected": 0.0015529401134699583,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9122215509414673,
      "logits/rejected": -1.9168663024902344,
      "logps/chosen": -31.424020767211914,
      "logps/rejected": -32.784080505371094,
      "loss": 0.3157,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.16403506696224213,
      "rewards/margins": 0.1578713059425354,
      "rewards/rejected": 0.0061637843027710915,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.237051010131836,
      "eval_logits/rejected": -2.2321863174438477,
      "eval_logps/chosen": -34.018070220947266,
      "eval_logps/rejected": -37.51838684082031,
      "eval_loss": 0.3716273605823517,
      "eval_rewards/accuracies": 0.5245016813278198,
      "eval_rewards/chosen": 0.014834923669695854,
      "eval_rewards/margins": 0.016425320878624916,
      "eval_rewards/rejected": -0.0015903981402516365,
      "eval_runtime": 145.5151,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.296,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.023789882659912,
      "logits/rejected": -2.034484386444092,
      "logps/chosen": -31.767370223999023,
      "logps/rejected": -33.890621185302734,
      "loss": 0.275,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.18354059755802155,
      "rewards/margins": 0.20867136120796204,
      "rewards/rejected": -0.025130782276391983,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.916685700416565,
      "logits/rejected": -1.9314892292022705,
      "logps/chosen": -29.956628799438477,
      "logps/rejected": -31.564035415649414,
      "loss": 0.2958,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13942097127437592,
      "rewards/margins": 0.14469322562217712,
      "rewards/rejected": -0.005272268317639828,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9737945795059204,
      "logits/rejected": -1.9777710437774658,
      "logps/chosen": -33.19129180908203,
      "logps/rejected": -31.5566463470459,
      "loss": 0.287,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.20355579257011414,
      "rewards/margins": 0.21016716957092285,
      "rewards/rejected": -0.0066113718785345554,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9743419885635376,
      "logits/rejected": -1.9523779153823853,
      "logps/chosen": -33.9401969909668,
      "logps/rejected": -35.008758544921875,
      "loss": 0.3185,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1921955645084381,
      "rewards/margins": 0.24436470866203308,
      "rewards/rejected": -0.052169155329465866,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0161709785461426,
      "logits/rejected": -2.0128414630889893,
      "logps/chosen": -32.74829864501953,
      "logps/rejected": -36.236392974853516,
      "loss": 0.2847,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.15909257531166077,
      "rewards/margins": 0.1694144755601883,
      "rewards/rejected": -0.010321905836462975,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8847742080688477,
      "logits/rejected": -1.8823268413543701,
      "logps/chosen": -34.01182174682617,
      "logps/rejected": -35.481346130371094,
      "loss": 0.3073,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.1589193046092987,
      "rewards/margins": 0.13797220587730408,
      "rewards/rejected": 0.020947108045220375,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8693536520004272,
      "logits/rejected": -1.866838812828064,
      "logps/chosen": -34.20549392700195,
      "logps/rejected": -31.726673126220703,
      "loss": 0.2733,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.17289286851882935,
      "rewards/margins": 0.1327010840177536,
      "rewards/rejected": 0.040191780775785446,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9730831384658813,
      "logits/rejected": -1.962480902671814,
      "logps/chosen": -35.06049728393555,
      "logps/rejected": -31.792781829833984,
      "loss": 0.2523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2665289342403412,
      "rewards/margins": 0.22314274311065674,
      "rewards/rejected": 0.04338619112968445,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0680813789367676,
      "logits/rejected": -2.053079128265381,
      "logps/chosen": -30.738479614257812,
      "logps/rejected": -32.61243438720703,
      "loss": 0.3194,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.16707859933376312,
      "rewards/margins": 0.13161785900592804,
      "rewards/rejected": 0.035460732877254486,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9398882389068604,
      "logits/rejected": -1.9373395442962646,
      "logps/chosen": -32.6181640625,
      "logps/rejected": -30.843700408935547,
      "loss": 0.2156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2780519425868988,
      "rewards/margins": 0.3022567331790924,
      "rewards/rejected": -0.024204757064580917,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2364187240600586,
      "eval_logits/rejected": -2.2315518856048584,
      "eval_logps/chosen": -34.01428985595703,
      "eval_logps/rejected": -37.496952056884766,
      "eval_loss": 0.38450533151626587,
      "eval_rewards/accuracies": 0.49335551261901855,
      "eval_rewards/chosen": 0.018236981704831123,
      "eval_rewards/margins": 0.0005384809919632971,
      "eval_rewards/rejected": 0.01769850216805935,
      "eval_runtime": 145.7485,
      "eval_samples_per_second": 2.353,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.922579050064087,
      "logits/rejected": -1.9193273782730103,
      "logps/chosen": -31.345911026000977,
      "logps/rejected": -33.72126007080078,
      "loss": 0.2917,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.23169513046741486,
      "rewards/margins": 0.21289470791816711,
      "rewards/rejected": 0.018800420686602592,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.9754743576049805,
      "logits/rejected": -1.9631853103637695,
      "logps/chosen": -34.408077239990234,
      "logps/rejected": -33.58232879638672,
      "loss": 0.2394,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1691979616880417,
      "rewards/margins": 0.20833876729011536,
      "rewards/rejected": -0.03914082050323486,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -2.0105607509613037,
      "logits/rejected": -2.009115219116211,
      "logps/chosen": -33.31591033935547,
      "logps/rejected": -32.47368621826172,
      "loss": 0.2769,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.15289874374866486,
      "rewards/margins": 0.14759239554405212,
      "rewards/rejected": 0.005306343547999859,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.096872091293335,
      "logits/rejected": -2.0811073780059814,
      "logps/chosen": -33.87510681152344,
      "logps/rejected": -33.06427764892578,
      "loss": 0.2792,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.27385497093200684,
      "rewards/margins": 0.2254684418439865,
      "rewards/rejected": 0.04838654398918152,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.969496488571167,
      "logits/rejected": -1.9685735702514648,
      "logps/chosen": -32.98945999145508,
      "logps/rejected": -32.4643440246582,
      "loss": 0.2958,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.26337358355522156,
      "rewards/margins": 0.26264840364456177,
      "rewards/rejected": 0.0007252089562825859,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9255645275115967,
      "logits/rejected": -1.9359004497528076,
      "logps/chosen": -32.013362884521484,
      "logps/rejected": -35.26326370239258,
      "loss": 0.3425,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1721937507390976,
      "rewards/margins": 0.14924712479114532,
      "rewards/rejected": 0.022946633398532867,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.0643956661224365,
      "logits/rejected": -2.057886838912964,
      "logps/chosen": -33.48772430419922,
      "logps/rejected": -29.191638946533203,
      "loss": 0.298,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14782151579856873,
      "rewards/margins": 0.11651048809289932,
      "rewards/rejected": 0.03131101652979851,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9235725402832031,
      "logits/rejected": -1.9257465600967407,
      "logps/chosen": -33.965919494628906,
      "logps/rejected": -30.839218139648438,
      "loss": 0.2616,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.24581687152385712,
      "rewards/margins": 0.2157471626996994,
      "rewards/rejected": 0.03006969951093197,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.31703355428460356,
      "train_runtime": 3251.5033,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}