{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.866518497467041,
      "logits/rejected": -1.8708432912826538,
      "logps/chosen": -36.97861862182617,
      "logps/rejected": -33.65688705444336,
      "loss": 0.9616,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.01958908699452877,
      "rewards/margins": 0.038356635719537735,
      "rewards/rejected": -0.018767546862363815,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.997629165649414,
      "logits/rejected": -2.0002758502960205,
      "logps/chosen": -29.631610870361328,
      "logps/rejected": -29.059566497802734,
      "loss": 0.9953,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00741056352853775,
      "rewards/margins": 0.004696969874203205,
      "rewards/rejected": 0.002713592257350683,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.9196974039077759,
      "logits/rejected": -1.9170036315917969,
      "logps/chosen": -31.402713775634766,
      "logps/rejected": -33.229434967041016,
      "loss": 0.9829,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.009367194958031178,
      "rewards/margins": 0.01705181784927845,
      "rewards/rejected": -0.007684621028602123,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.0161232948303223,
      "logits/rejected": -2.0073835849761963,
      "logps/chosen": -32.559471130371094,
      "logps/rejected": -32.493953704833984,
      "loss": 1.001,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.012133684009313583,
      "rewards/margins": -0.0009662110242061317,
      "rewards/rejected": 0.013099895790219307,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.8646795749664307,
      "logits/rejected": -1.8538821935653687,
      "logps/chosen": -33.56920623779297,
      "logps/rejected": -35.44689178466797,
      "loss": 1.012,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.008206313475966454,
      "rewards/margins": -0.012022162787616253,
      "rewards/rejected": 0.003815849544480443,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9452946186065674,
      "logits/rejected": -1.9472544193267822,
      "logps/chosen": -32.58448791503906,
      "logps/rejected": -33.194095611572266,
      "loss": 0.9731,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010985399596393108,
      "rewards/margins": 0.02688519097864628,
      "rewards/rejected": -0.015899788588285446,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.079716682434082,
      "logits/rejected": -2.084702730178833,
      "logps/chosen": -33.99287033081055,
      "logps/rejected": -36.58139419555664,
      "loss": 0.9943,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0030690780840814114,
      "rewards/margins": 0.00569509994238615,
      "rewards/rejected": -0.008764175698161125,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.9423043727874756,
      "logits/rejected": -1.9454625844955444,
      "logps/chosen": -34.40251922607422,
      "logps/rejected": -34.589447021484375,
      "loss": 0.9844,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.017706653103232384,
      "rewards/margins": 0.015634063631296158,
      "rewards/rejected": 0.0020725864451378584,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.950506567955017,
      "logits/rejected": -1.955021619796753,
      "logps/chosen": -32.44976806640625,
      "logps/rejected": -32.36195373535156,
      "loss": 0.9855,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.012356681749224663,
      "rewards/margins": 0.01448807679116726,
      "rewards/rejected": -0.0021313955076038837,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0489630699157715,
      "logits/rejected": -2.046970844268799,
      "logps/chosen": -32.250099182128906,
      "logps/rejected": -31.2716121673584,
      "loss": 1.0027,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.006484628655016422,
      "rewards/margins": -0.002702844562008977,
      "rewards/rejected": -0.003781783627346158,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.244150400161743,
      "eval_logits/rejected": -2.239272117614746,
      "eval_logps/chosen": -34.02294921875,
      "eval_logps/rejected": -37.49901580810547,
      "eval_loss": 1.0042858123779297,
      "eval_rewards/accuracies": 0.4958471655845642,
      "eval_rewards/chosen": 0.00812594871968031,
      "eval_rewards/margins": -0.004197616130113602,
      "eval_rewards/rejected": 0.012323564849793911,
      "eval_runtime": 146.0748,
      "eval_samples_per_second": 2.348,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.006051540374756,
      "logits/rejected": -2.003640651702881,
      "logps/chosen": -33.229209899902344,
      "logps/rejected": -34.01814651489258,
      "loss": 0.9976,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.010226872749626637,
      "rewards/margins": 0.0023667856585234404,
      "rewards/rejected": 0.007860084064304829,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.016700506210327,
      "logits/rejected": -2.0083200931549072,
      "logps/chosen": -32.47074508666992,
      "logps/rejected": -32.16743087768555,
      "loss": 1.0139,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.018086189404129982,
      "rewards/margins": -0.01393910776823759,
      "rewards/rejected": -0.004147083964198828,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.046416759490967,
      "logits/rejected": -2.0383784770965576,
      "logps/chosen": -30.48056411743164,
      "logps/rejected": -32.03020477294922,
      "loss": 1.0194,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.003392170649021864,
      "rewards/margins": -0.019355837255716324,
      "rewards/rejected": 0.015963664278388023,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.977251648902893,
      "logits/rejected": -1.9875211715698242,
      "logps/chosen": -31.414047241210938,
      "logps/rejected": -32.53428268432617,
      "loss": 0.9965,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.003930580802261829,
      "rewards/margins": 0.0035118304658681154,
      "rewards/rejected": 0.0004187510348856449,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.8907463550567627,
      "logits/rejected": -1.8918399810791016,
      "logps/chosen": -34.17836380004883,
      "logps/rejected": -34.76056671142578,
      "loss": 0.9877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010030902922153473,
      "rewards/margins": 0.012347406707704067,
      "rewards/rejected": -0.002316503319889307,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.9423500299453735,
      "logits/rejected": -1.938869833946228,
      "logps/chosen": -36.138328552246094,
      "logps/rejected": -32.716270446777344,
      "loss": 0.9812,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023342430591583252,
      "rewards/margins": 0.018807319924235344,
      "rewards/rejected": 0.004535110667347908,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.042358160018921,
      "logits/rejected": -2.034965753555298,
      "logps/chosen": -33.78316879272461,
      "logps/rejected": -31.378530502319336,
      "loss": 0.971,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010285439901053905,
      "rewards/margins": 0.028961649164557457,
      "rewards/rejected": -0.018676208332180977,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.0474698543548584,
      "logits/rejected": -2.0527453422546387,
      "logps/chosen": -32.541080474853516,
      "logps/rejected": -32.49051284790039,
      "loss": 0.9936,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0025936015881597996,
      "rewards/margins": 0.00635246466845274,
      "rewards/rejected": -0.0037588649429380894,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.048386573791504,
      "logits/rejected": -2.045606851577759,
      "logps/chosen": -31.502283096313477,
      "logps/rejected": -31.347753524780273,
      "loss": 0.9887,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.00933015439659357,
      "rewards/margins": 0.011346332728862762,
      "rewards/rejected": -0.02067648433148861,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.9189811944961548,
      "logits/rejected": -1.9236608743667603,
      "logps/chosen": -31.6005859375,
      "logps/rejected": -32.77867889404297,
      "loss": 1.0046,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.003988111391663551,
      "rewards/margins": -0.004586963914334774,
      "rewards/rejected": 0.0085750762373209,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.24337100982666,
      "eval_logits/rejected": -2.238497257232666,
      "eval_logps/chosen": -34.03440856933594,
      "eval_logps/rejected": -37.50404357910156,
      "eval_loss": 1.0087451934814453,
      "eval_rewards/accuracies": 0.46677741408348083,
      "eval_rewards/chosen": 0.00010354903497500345,
      "eval_rewards/margins": -0.008702815510332584,
      "eval_rewards/rejected": 0.008806362748146057,
      "eval_runtime": 145.922,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0320065021514893,
      "logits/rejected": -2.0426933765411377,
      "logps/chosen": -31.95755386352539,
      "logps/rejected": -33.87141036987305,
      "loss": 0.9843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00962692778557539,
      "rewards/margins": 0.015725601464509964,
      "rewards/rejected": -0.006098673213273287,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.9255682229995728,
      "logits/rejected": -1.9404188394546509,
      "logps/chosen": -30.090320587158203,
      "logps/rejected": -31.563533782958984,
      "loss": 0.9814,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014854127541184425,
      "rewards/margins": 0.018603894859552383,
      "rewards/rejected": -0.003749764757230878,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.9833379983901978,
      "logits/rejected": -1.987309455871582,
      "logps/chosen": -33.385955810546875,
      "logps/rejected": -31.533649444580078,
      "loss": 0.9889,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.02206074632704258,
      "rewards/margins": 0.011102994903922081,
      "rewards/rejected": 0.010957752354443073,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.9829800128936768,
      "logits/rejected": -1.9610090255737305,
      "logps/chosen": -34.147132873535156,
      "logps/rejected": -34.971717834472656,
      "loss": 0.9807,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.004631907679140568,
      "rewards/margins": 0.019280528649687767,
      "rewards/rejected": -0.0146486209705472,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.024630069732666,
      "logits/rejected": -2.021322011947632,
      "logps/chosen": -32.90209197998047,
      "logps/rejected": -36.2330207824707,
      "loss": 0.9782,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.016082406044006348,
      "rewards/margins": 0.021750962361693382,
      "rewards/rejected": -0.005668557249009609,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.8910694122314453,
      "logits/rejected": -1.888628602027893,
      "logps/chosen": -34.16587448120117,
      "logps/rejected": -35.542091369628906,
      "loss": 0.958,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015768656507134438,
      "rewards/margins": 0.04199688881635666,
      "rewards/rejected": -0.026228230446577072,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.8759692907333374,
      "logits/rejected": -1.8734344244003296,
      "logps/chosen": -34.40534973144531,
      "logps/rejected": -31.75263023376465,
      "loss": 1.0185,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005422012414783239,
      "rewards/margins": -0.01851092278957367,
      "rewards/rejected": 0.013088913634419441,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.979745626449585,
      "logits/rejected": -1.9691104888916016,
      "logps/chosen": -35.3358268737793,
      "logps/rejected": -31.83913803100586,
      "loss": 0.9867,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014568204991519451,
      "rewards/margins": 0.013271312229335308,
      "rewards/rejected": 0.0012968936935067177,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.075892210006714,
      "logits/rejected": -2.060863494873047,
      "logps/chosen": -30.9158935546875,
      "logps/rejected": -32.63315963745117,
      "loss": 1.0073,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.005761646665632725,
      "rewards/margins": -0.007310979068279266,
      "rewards/rejected": 0.013072624802589417,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.9466651678085327,
      "logits/rejected": -1.9441168308258057,
      "logps/chosen": -32.88911056518555,
      "logps/rejected": -30.80002784729004,
      "loss": 0.9851,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.026600470766425133,
      "rewards/margins": 0.014853144064545631,
      "rewards/rejected": 0.011747325770556927,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2438066005706787,
      "eval_logits/rejected": -2.238919258117676,
      "eval_logps/chosen": -34.02598190307617,
      "eval_logps/rejected": -37.4908561706543,
      "eval_loss": 1.0122511386871338,
      "eval_rewards/accuracies": 0.45514950156211853,
      "eval_rewards/chosen": 0.005999694112688303,
      "eval_rewards/margins": -0.01203568559139967,
      "eval_rewards/rejected": 0.01803538203239441,
      "eval_runtime": 145.863,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.9287000894546509,
      "logits/rejected": -1.9254179000854492,
      "logps/chosen": -31.583938598632812,
      "logps/rejected": -33.75525665283203,
      "loss": 0.9772,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.013588106259703636,
      "rewards/margins": 0.022759849205613136,
      "rewards/rejected": -0.009171743877232075,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.9809232950210571,
      "logits/rejected": -1.9685947895050049,
      "logps/chosen": -34.56711196899414,
      "logps/rejected": -33.54813003540039,
      "loss": 0.9732,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.020275089889764786,
      "rewards/margins": 0.026777852326631546,
      "rewards/rejected": -0.006502761039882898,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.0157439708709717,
      "logits/rejected": -2.014296531677246,
      "logps/chosen": -33.48149108886719,
      "logps/rejected": -32.470550537109375,
      "loss": 1.0033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0030108082573860884,
      "rewards/margins": -0.0033085450995713472,
      "rewards/rejected": 0.00631935428828001,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.1027872562408447,
      "logits/rejected": -2.086984157562256,
      "logps/chosen": -34.1781120300293,
      "logps/rejected": -33.094017028808594,
      "loss": 1.0159,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0008956976234912872,
      "rewards/margins": -0.015917152166366577,
      "rewards/rejected": 0.016812851652503014,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.9746129512786865,
      "logits/rejected": -1.9736511707305908,
      "logps/chosen": -33.25445556640625,
      "logps/rejected": -32.4587287902832,
      "loss": 0.9851,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019348327070474625,
      "rewards/margins": 0.014853959903120995,
      "rewards/rejected": 0.004494364373385906,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9303343296051025,
      "logits/rejected": -1.940706491470337,
      "logps/chosen": -32.189186096191406,
      "logps/rejected": -35.288429260253906,
      "loss": 0.9894,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.010855492204427719,
      "rewards/margins": 0.010628147050738335,
      "rewards/rejected": 0.00022734627418685704,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.0694239139556885,
      "logits/rejected": -2.0628561973571777,
      "logps/chosen": -33.65647506713867,
      "logps/rejected": -29.229717254638672,
      "loss": 1.0008,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0031533471774309874,
      "rewards/margins": -0.0008490110631100833,
      "rewards/rejected": -0.0023043369874358177,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.9291013479232788,
      "logits/rejected": -1.9312855005264282,
      "logps/chosen": -34.26526641845703,
      "logps/rejected": -30.913259506225586,
      "loss": 0.9899,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.01835118792951107,
      "rewards/margins": 0.010092146694660187,
      "rewards/rejected": -0.028443332761526108,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.9909477320584383,
      "train_runtime": 3254.3492,
      "train_samples_per_second": 0.946,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}