{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8667426109313965,
      "logits/rejected": -1.8710602521896362,
      "logps/chosen": -36.991912841796875,
      "logps/rejected": -33.67206954956055,
      "loss": 0.9773,
      "rewards/accuracies": 0.5694444179534912,
      "rewards/chosen": 0.0058750128373503685,
      "rewards/margins": 0.022671451792120934,
      "rewards/rejected": -0.01679643802344799,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9977442026138306,
      "logits/rejected": -2.0003952980041504,
      "logps/chosen": -29.659366607666016,
      "logps/rejected": -29.05437660217285,
      "loss": 1.0105,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.006868218071758747,
      "rewards/margins": -0.010493903420865536,
      "rewards/rejected": 0.003625686513260007,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.920693039894104,
      "logits/rejected": -1.91802179813385,
      "logps/chosen": -31.39971351623535,
      "logps/rejected": -33.21495819091797,
      "loss": 0.9948,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006551843136548996,
      "rewards/margins": 0.0051523735746741295,
      "rewards/rejected": 0.001399471191689372,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.018057107925415,
      "logits/rejected": -2.0093047618865967,
      "logps/chosen": -32.565284729003906,
      "logps/rejected": -32.50053405761719,
      "loss": 1.0002,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0046076299622654915,
      "rewards/margins": -0.00024683662923052907,
      "rewards/rejected": 0.004854466766119003,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8627817630767822,
      "logits/rejected": -1.851999044418335,
      "logps/chosen": -33.549964904785156,
      "logps/rejected": -35.44340896606445,
      "loss": 1.0006,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0030060340650379658,
      "rewards/margins": -0.0005688609671778977,
      "rewards/rejected": 0.003574896603822708,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9416770935058594,
      "logits/rejected": -1.9436094760894775,
      "logps/chosen": -32.53351593017578,
      "logps/rejected": -33.217529296875,
      "loss": 0.9561,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.026666466146707535,
      "rewards/margins": 0.04512657970190048,
      "rewards/rejected": -0.018460111692547798,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.072779655456543,
      "logits/rejected": -2.077756881713867,
      "logps/chosen": -34.002342224121094,
      "logps/rejected": -36.633216857910156,
      "loss": 0.9798,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0055419160053133965,
      "rewards/margins": 0.020196745172142982,
      "rewards/rejected": -0.025738662108778954,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9332294464111328,
      "logits/rejected": -1.9363486766815186,
      "logps/chosen": -34.33064651489258,
      "logps/rejected": -34.64745330810547,
      "loss": 0.9391,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03886651247739792,
      "rewards/margins": 0.06088464334607124,
      "rewards/rejected": -0.022018127143383026,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9409536123275757,
      "logits/rejected": -1.9454677104949951,
      "logps/chosen": -32.381752014160156,
      "logps/rejected": -32.35805892944336,
      "loss": 0.9661,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0342683270573616,
      "rewards/margins": 0.033929694443941116,
      "rewards/rejected": 0.0003386303724255413,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.0382819175720215,
      "logits/rejected": -2.036304473876953,
      "logps/chosen": -32.15763473510742,
      "logps/rejected": -31.302764892578125,
      "loss": 0.9521,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.033279187977313995,
      "rewards/margins": 0.047900475561618805,
      "rewards/rejected": -0.014621290378272533,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2327499389648438,
      "eval_logits/rejected": -2.227902889251709,
      "eval_logps/chosen": -34.038063049316406,
      "eval_logps/rejected": -37.54012680053711,
      "eval_loss": 0.9920137524604797,
      "eval_rewards/accuracies": 0.5361295938491821,
      "eval_rewards/chosen": -0.0014053798513486981,
      "eval_rewards/margins": 0.007998107932507992,
      "eval_rewards/rejected": -0.009403487667441368,
      "eval_runtime": 146.1087,
      "eval_samples_per_second": 2.348,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9931761026382446,
      "logits/rejected": -1.9907801151275635,
      "logps/chosen": -33.120845794677734,
      "logps/rejected": -34.02234649658203,
      "loss": 0.9536,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04918808490037918,
      "rewards/margins": 0.046377379447221756,
      "rewards/rejected": 0.0028106991667300463,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.004035472869873,
      "logits/rejected": -1.9957201480865479,
      "logps/chosen": -32.31081771850586,
      "logps/rejected": -32.129127502441406,
      "loss": 0.9593,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05363499000668526,
      "rewards/margins": 0.04068244248628616,
      "rewards/rejected": 0.012952548451721668,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0312001705169678,
      "logits/rejected": -2.023239850997925,
      "logps/chosen": -30.317270278930664,
      "logps/rejected": -32.05809783935547,
      "loss": 0.9384,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06337819993495941,
      "rewards/margins": 0.06541591882705688,
      "rewards/rejected": -0.0020377314649522305,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9611848592758179,
      "logits/rejected": -1.9714059829711914,
      "logps/chosen": -31.235523223876953,
      "logps/rejected": -32.55936813354492,
      "loss": 0.9165,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07365532219409943,
      "rewards/margins": 0.08345074951648712,
      "rewards/rejected": -0.009795431979000568,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8718490600585938,
      "logits/rejected": -1.8730093240737915,
      "logps/chosen": -33.873046875,
      "logps/rejected": -34.80980682373047,
      "loss": 0.8563,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.12785716354846954,
      "rewards/margins": 0.1488770991563797,
      "rewards/rejected": -0.021019931882619858,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9235217571258545,
      "logits/rejected": -1.9201278686523438,
      "logps/chosen": -36.01029968261719,
      "logps/rejected": -32.690834045410156,
      "loss": 0.9482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06455211341381073,
      "rewards/margins": 0.05178683251142502,
      "rewards/rejected": 0.012765283696353436,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0244345664978027,
      "logits/rejected": -2.0171027183532715,
      "logps/chosen": -33.474937438964844,
      "logps/rejected": -31.42409324645996,
      "loss": 0.8419,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.12916772067546844,
      "rewards/margins": 0.15806543827056885,
      "rewards/rejected": -0.028897713869810104,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0310873985290527,
      "logits/rejected": -2.036334276199341,
      "logps/chosen": -32.2062873840332,
      "logps/rejected": -32.42302703857422,
      "loss": 0.8919,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1354007124900818,
      "rewards/margins": 0.11055666208267212,
      "rewards/rejected": 0.024844054132699966,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0314042568206787,
      "logits/rejected": -2.02862286567688,
      "logps/chosen": -31.278457641601562,
      "logps/rejected": -31.348251342773438,
      "loss": 0.9038,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0841989517211914,
      "rewards/margins": 0.09621445834636688,
      "rewards/rejected": -0.012015508487820625,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9020229578018188,
      "logits/rejected": -1.906660795211792,
      "logps/chosen": -31.28672218322754,
      "logps/rejected": -32.84270477294922,
      "loss": 0.8515,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.12782469391822815,
      "rewards/margins": 0.14853176474571228,
      "rewards/rejected": -0.020707078278064728,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.22743558883667,
      "eval_logits/rejected": -2.2226204872131348,
      "eval_logps/chosen": -34.04505920410156,
      "eval_logps/rejected": -37.573036193847656,
      "eval_loss": 0.9816663265228271,
      "eval_rewards/accuracies": 0.559385359287262,
      "eval_rewards/chosen": -0.004204160068184137,
      "eval_rewards/margins": 0.018361244350671768,
      "eval_rewards/rejected": -0.022565403953194618,
      "eval_runtime": 145.491,
      "eval_samples_per_second": 2.358,
      "eval_steps_per_second": 0.296,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.0146515369415283,
      "logits/rejected": -2.025285005569458,
      "logps/chosen": -31.747217178344727,
      "logps/rejected": -33.96654510498047,
      "loss": 0.8755,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.08963600546121597,
      "rewards/margins": 0.13117524981498718,
      "rewards/rejected": -0.04153924435377121,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9072997570037842,
      "logits/rejected": -1.922041654586792,
      "logps/chosen": -29.81575584411621,
      "logps/rejected": -31.636306762695312,
      "loss": 0.8509,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.11831430345773697,
      "rewards/margins": 0.14956556260585785,
      "rewards/rejected": -0.03125125169754028,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9637253284454346,
      "logits/rejected": -1.9677000045776367,
      "logps/chosen": -33.084877014160156,
      "logps/rejected": -31.639108657836914,
      "loss": 0.8476,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.13303686678409576,
      "rewards/margins": 0.16895917057991028,
      "rewards/rejected": -0.03592229634523392,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9612106084823608,
      "logits/rejected": -1.9394094944000244,
      "logps/chosen": -33.84415817260742,
      "logps/rejected": -35.132205963134766,
      "loss": 0.8154,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1238357201218605,
      "rewards/margins": 0.19639968872070312,
      "rewards/rejected": -0.07256398350000381,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.002776861190796,
      "logits/rejected": -1.999464988708496,
      "logps/chosen": -32.74934387207031,
      "logps/rejected": -36.28064727783203,
      "loss": 0.9096,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0702909380197525,
      "rewards/margins": 0.09257940202951431,
      "rewards/rejected": -0.02228846587240696,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8692924976348877,
      "logits/rejected": -1.8668625354766846,
      "logps/chosen": -33.982818603515625,
      "logps/rejected": -35.535736083984375,
      "loss": 0.9053,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.08223171532154083,
      "rewards/margins": 0.09467832744121552,
      "rewards/rejected": -0.012446624226868153,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8543907403945923,
      "logits/rejected": -1.8520078659057617,
      "logps/chosen": -34.20638656616211,
      "logps/rejected": -31.873388290405273,
      "loss": 0.8923,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.07648433744907379,
      "rewards/margins": 0.11730633676052094,
      "rewards/rejected": -0.040821999311447144,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9576927423477173,
      "logits/rejected": -1.9472328424453735,
      "logps/chosen": -34.999000549316406,
      "logps/rejected": -31.88382911682129,
      "loss": 0.8398,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.1430555135011673,
      "rewards/margins": 0.16019006073474884,
      "rewards/rejected": -0.017134560272097588,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0523831844329834,
      "logits/rejected": -2.037504196166992,
      "logps/chosen": -30.712234497070312,
      "logps/rejected": -32.659637451171875,
      "loss": 0.9146,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.08475608378648758,
      "rewards/margins": 0.08787757158279419,
      "rewards/rejected": -0.0031214915215969086,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9236303567886353,
      "logits/rejected": -1.9211324453353882,
      "logps/chosen": -32.370330810546875,
      "logps/rejected": -30.952068328857422,
      "loss": 0.7473,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2227112501859665,
      "rewards/margins": 0.27681466937065125,
      "rewards/rejected": -0.054103411734104156,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.223449468612671,
      "eval_logits/rejected": -2.2186241149902344,
      "eval_logps/chosen": -34.09219741821289,
      "eval_logps/rejected": -37.61648178100586,
      "eval_loss": 0.9830461740493774,
      "eval_rewards/accuracies": 0.5390365719795227,
      "eval_rewards/chosen": -0.023058738559484482,
      "eval_rewards/margins": 0.016884688287973404,
      "eval_rewards/rejected": -0.03994342312216759,
      "eval_runtime": 145.6848,
      "eval_samples_per_second": 2.354,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.908721923828125,
      "logits/rejected": -1.9054863452911377,
      "logps/chosen": -31.30953025817871,
      "logps/rejected": -33.8243408203125,
      "loss": 0.8639,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.1175277829170227,
      "rewards/margins": 0.15040358901023865,
      "rewards/rejected": -0.032875802367925644,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.958335518836975,
      "logits/rejected": -1.946170449256897,
      "logps/chosen": -34.308921813964844,
      "logps/rejected": -33.70027542114258,
      "loss": 0.8275,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11486158519983292,
      "rewards/margins": 0.17943526804447174,
      "rewards/rejected": -0.06457368284463882,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -1.9931294918060303,
      "logits/rejected": -1.9917207956314087,
      "logps/chosen": -33.1971321105957,
      "logps/rejected": -32.582763671875,
      "loss": 0.8474,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.11546528339385986,
      "rewards/margins": 0.15673772990703583,
      "rewards/rejected": -0.04127243906259537,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.0800864696502686,
      "logits/rejected": -2.064441204071045,
      "logps/chosen": -33.79554748535156,
      "logps/rejected": -33.09272384643555,
      "loss": 0.8566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.15353715419769287,
      "rewards/margins": 0.14341332018375397,
      "rewards/rejected": 0.010123846121132374,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.9522091150283813,
      "logits/rejected": -1.9513927698135376,
      "logps/chosen": -32.85493469238281,
      "logps/rejected": -32.53038787841797,
      "loss": 0.818,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.170864075422287,
      "rewards/margins": 0.19696101546287537,
      "rewards/rejected": -0.02609694004058838,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9075958728790283,
      "logits/rejected": -1.9178472757339478,
      "logps/chosen": -31.86941909790039,
      "logps/rejected": -35.343692779541016,
      "loss": 0.8461,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1341095268726349,
      "rewards/margins": 0.1560816764831543,
      "rewards/rejected": -0.021972158923745155,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.047290325164795,
      "logits/rejected": -2.0408453941345215,
      "logps/chosen": -33.33329772949219,
      "logps/rejected": -29.256567001342773,
      "loss": 0.8605,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.12746797502040863,
      "rewards/margins": 0.1395258903503418,
      "rewards/rejected": -0.012057906948029995,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9072849750518799,
      "logits/rejected": -1.909480333328247,
      "logps/chosen": -33.89991760253906,
      "logps/rejected": -31.013565063476562,
      "loss": 0.8133,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.13565407693386078,
      "rewards/margins": 0.1920291632413864,
      "rewards/rejected": -0.05637507513165474,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.8990784768934373,
      "train_runtime": 3250.3805,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}