{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.8663256168365479,
      "logits/rejected": -1.8706417083740234,
      "logps/chosen": -36.98673629760742,
      "logps/rejected": -33.66697311401367,
      "loss": 0.9994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00019861094187945127,
      "rewards/margins": 0.0005675733555108309,
      "rewards/rejected": -0.0003689624136313796,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.997527837753296,
      "logits/rejected": -2.0001606941223145,
      "logps/chosen": -29.65478515625,
      "logps/rejected": -29.070556640625,
      "loss": 1.0001,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.00012590530968736857,
      "rewards/margins": -5.477964805322699e-05,
      "rewards/rejected": -7.112567982403561e-05,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.9195702075958252,
      "logits/rejected": -1.9168812036514282,
      "logps/chosen": -31.421009063720703,
      "logps/rejected": -33.225616455078125,
      "loss": 1.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -4.912544318358414e-05,
      "rewards/margins": 2.244002462248318e-05,
      "rewards/rejected": -7.15654605301097e-05,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.0165441036224365,
      "logits/rejected": -2.0078024864196777,
      "logps/chosen": -32.57805252075195,
      "logps/rejected": -32.504085540771484,
      "loss": 1.0001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.2443718333088327e-05,
      "rewards/margins": -9.829048212850466e-05,
      "rewards/rejected": 8.584677561884746e-05,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.8641563653945923,
      "logits/rejected": -1.8533931970596313,
      "logps/chosen": -33.555824279785156,
      "logps/rejected": -35.42582321166992,
      "loss": 1.0002,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 1.6570575098739937e-05,
      "rewards/margins": -0.0002486651937942952,
      "rewards/rejected": 0.000265235808910802,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9452263116836548,
      "logits/rejected": -1.9471622705459595,
      "logps/chosen": -32.5916633605957,
      "logps/rejected": -33.1885871887207,
      "loss": 0.9997,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 8.520439587300643e-05,
      "rewards/margins": 0.00025727329193614423,
      "rewards/rejected": -0.0001720689469948411,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.0798425674438477,
      "logits/rejected": -2.08483624458313,
      "logps/chosen": -33.98626708984375,
      "logps/rejected": -36.57551956176758,
      "loss": 0.9999,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 2.217638393631205e-05,
      "rewards/margins": 8.864814299158752e-05,
      "rewards/rejected": -6.647173722740263e-05,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.942161202430725,
      "logits/rejected": -1.9453051090240479,
      "logps/chosen": -34.400901794433594,
      "logps/rejected": -34.57304382324219,
      "loss": 0.9999,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00026908880681730807,
      "rewards/margins": 7.544312393292785e-05,
      "rewards/rejected": 0.000193645668332465,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.9505256414413452,
      "logits/rejected": -1.9550361633300781,
      "logps/chosen": -32.452125549316406,
      "logps/rejected": -32.35675811767578,
      "loss": 0.9999,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.00015296679339371622,
      "rewards/margins": 0.0001315164554398507,
      "rewards/rejected": 2.145032885891851e-05,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0490074157714844,
      "logits/rejected": -2.0470099449157715,
      "logps/chosen": -32.25472640991211,
      "logps/rejected": -31.2731990814209,
      "loss": 1.0001,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.00013893461436964571,
      "rewards/margins": -6.904623296577483e-05,
      "rewards/rejected": -6.98883886798285e-05,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2441036701202393,
      "eval_logits/rejected": -2.239219903945923,
      "eval_logps/chosen": -34.02959442138672,
      "eval_logps/rejected": -37.50800704956055,
      "eval_loss": 1.0000391006469727,
      "eval_rewards/accuracies": 0.5099667906761169,
      "eval_rewards/chosen": 4.9558610044186935e-05,
      "eval_rewards/margins": -3.6566227208822966e-05,
      "eval_rewards/rejected": 8.61248336150311e-05,
      "eval_runtime": 145.9645,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.005286931991577,
      "logits/rejected": -2.0028929710388184,
      "logps/chosen": -33.22523498535156,
      "logps/rejected": -34.039817810058594,
      "loss": 0.9997,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00018582009943202138,
      "rewards/margins": 0.0002903031127061695,
      "rewards/rejected": -0.00010448300599819049,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.0164072513580322,
      "logits/rejected": -2.008033514022827,
      "logps/chosen": -32.473182678222656,
      "logps/rejected": -32.18474197387695,
      "loss": 1.0001,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.00028277214732952416,
      "rewards/margins": -5.0430990086169913e-05,
      "rewards/rejected": -0.00023234116088133305,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.0463318824768066,
      "logits/rejected": -2.03826642036438,
      "logps/chosen": -30.479427337646484,
      "logps/rejected": -32.030479431152344,
      "loss": 1.0003,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -3.712148100021295e-05,
      "rewards/margins": -0.00026237856945954263,
      "rewards/rejected": 0.0002252570993732661,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.9770317077636719,
      "logits/rejected": -1.987318754196167,
      "logps/chosen": -31.41189193725586,
      "logps/rejected": -32.552860260009766,
      "loss": 0.9997,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 7.772201206535101e-05,
      "rewards/margins": 0.000257515290286392,
      "rewards/rejected": -0.00017979330732487142,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.8904685974121094,
      "logits/rejected": -1.8915725946426392,
      "logps/chosen": -34.178749084472656,
      "logps/rejected": -34.7522087097168,
      "loss": 0.9999,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00013941082579549402,
      "rewards/margins": 8.894065831555054e-05,
      "rewards/rejected": 5.047018566983752e-05,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.9424184560775757,
      "logits/rejected": -1.9389402866363525,
      "logps/chosen": -36.14480209350586,
      "logps/rejected": -32.753517150878906,
      "loss": 0.9994,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0002687961678020656,
      "rewards/margins": 0.0005764620145782828,
      "rewards/rejected": -0.0003076658467762172,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.0420711040496826,
      "logits/rejected": -2.0346856117248535,
      "logps/chosen": -33.783363342285156,
      "logps/rejected": -31.349674224853516,
      "loss": 0.9999,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00014495299546979368,
      "rewards/margins": 0.00012318804510869086,
      "rewards/rejected": 2.1765008568763733e-05,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.0475425720214844,
      "logits/rejected": -2.052799940109253,
      "logps/chosen": -32.50774383544922,
      "logps/rejected": -32.48485565185547,
      "loss": 0.9996,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00037043425254523754,
      "rewards/margins": 0.0003675749758258462,
      "rewards/rejected": 2.859340838767821e-06,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.0484375953674316,
      "logits/rejected": -2.045671224594116,
      "logps/chosen": -31.50165367126465,
      "logps/rejected": -31.33974266052246,
      "loss": 0.9999,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00012700485240202397,
      "rewards/margins": 8.82898602867499e-05,
      "rewards/rejected": -0.00021529469813685864,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.9182907342910767,
      "logits/rejected": -1.9229600429534912,
      "logps/chosen": -31.5924072265625,
      "logps/rejected": -32.794044494628906,
      "loss": 0.9998,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.00013877781748306006,
      "rewards/margins": 0.0001699000713415444,
      "rewards/rejected": -3.112227204837836e-05,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2440481185913086,
      "eval_logits/rejected": -2.2391700744628906,
      "eval_logps/chosen": -34.02214813232422,
      "eval_logps/rejected": -37.503997802734375,
      "eval_loss": 1.0000054836273193,
      "eval_rewards/accuracies": 0.5045680999755859,
      "eval_rewards/chosen": 0.00012402697757352144,
      "eval_rewards/margins": -2.209126932939398e-06,
      "eval_rewards/rejected": 0.00012623611837625504,
      "eval_runtime": 145.6161,
      "eval_samples_per_second": 2.356,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0312139987945557,
      "logits/rejected": -2.041886568069458,
      "logps/chosen": -31.938335418701172,
      "logps/rejected": -33.87850570678711,
      "loss": 0.9995,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00032970920437946916,
      "rewards/margins": 0.0004878062754869461,
      "rewards/rejected": -0.00015809701289981604,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.925606369972229,
      "logits/rejected": -1.940452218055725,
      "logps/chosen": -30.110549926757812,
      "logps/rejected": -31.573543548583984,
      "loss": 0.9998,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 9.872082955553196e-06,
      "rewards/margins": 0.00016354399849660695,
      "rewards/rejected": -0.00015367190644610673,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.983096718788147,
      "logits/rejected": -1.9870779514312744,
      "logps/chosen": -33.37377166748047,
      "logps/rejected": -31.558422088623047,
      "loss": 0.9995,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0004369509988464415,
      "rewards/margins": 0.0005281810299493372,
      "rewards/rejected": -9.123003837885335e-05,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.9832874536514282,
      "logits/rejected": -1.9613001346588135,
      "logps/chosen": -34.15814971923828,
      "logps/rejected": -34.962947845458984,
      "loss": 0.9999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.397642987896688e-05,
      "rewards/margins": 7.76300803408958e-05,
      "rewards/rejected": -0.00012160651385784149,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.024568557739258,
      "logits/rejected": -2.021247386932373,
      "logps/chosen": -32.91596221923828,
      "logps/rejected": -36.22909164428711,
      "loss": 0.9999,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 9.10547751118429e-05,
      "rewards/margins": 0.00013272995420265943,
      "rewards/rejected": -4.1675208194646984e-05,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.891326904296875,
      "logits/rejected": -1.8888810873031616,
      "logps/chosen": -34.165008544921875,
      "logps/rejected": -35.511131286621094,
      "loss": 0.9997,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00023390687420032918,
      "rewards/margins": 0.00029896467458456755,
      "rewards/rejected": -6.505780766019598e-05,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.8756563663482666,
      "logits/rejected": -1.87309992313385,
      "logps/chosen": -34.39522171020508,
      "logps/rejected": -31.7664852142334,
      "loss": 1.0,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 2.3789130864315666e-05,
      "rewards/margins": -2.4663497242727317e-05,
      "rewards/rejected": 4.8452602641191334e-05,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.9794845581054688,
      "logits/rejected": -1.9688661098480225,
      "logps/chosen": -35.318504333496094,
      "logps/rejected": -31.839763641357422,
      "loss": 0.9996,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0003813380899373442,
      "rewards/margins": 0.0003690699231810868,
      "rewards/rejected": 1.2268178579688538e-05,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.075482130050659,
      "logits/rejected": -2.060464859008789,
      "logps/chosen": -30.921621322631836,
      "logps/rejected": -32.64299774169922,
      "loss": 1.0001,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 2.502280767657794e-05,
      "rewards/margins": -6.332028715405613e-05,
      "rewards/rejected": 8.834306936478242e-05,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.9463533163070679,
      "logits/rejected": -1.9438135623931885,
      "logps/chosen": -32.896881103515625,
      "logps/rejected": -30.816125869750977,
      "loss": 0.9997,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0003022877499461174,
      "rewards/margins": 0.0002954433439299464,
      "rewards/rejected": 6.84439146425575e-06,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.243636131286621,
      "eval_logits/rejected": -2.2387540340423584,
      "eval_logps/chosen": -34.02006530761719,
      "eval_logps/rejected": -37.50526428222656,
      "eval_loss": 0.9999700784683228,
      "eval_rewards/accuracies": 0.4987541437149048,
      "eval_rewards/chosen": 0.0001448561524739489,
      "eval_rewards/margins": 3.130395998596214e-05,
      "eval_rewards/rejected": 0.00011355219612596557,
      "eval_runtime": 145.6582,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.9288015365600586,
      "logits/rejected": -1.9255383014678955,
      "logps/chosen": -31.57794189453125,
      "logps/rejected": -33.73435592651367,
      "loss": 0.9998,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.000254074577242136,
      "rewards/margins": 0.00017612635565456003,
      "rewards/rejected": 7.794820703566074e-05,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.9808727502822876,
      "logits/rejected": -1.9685646295547485,
      "logps/chosen": -34.54682159423828,
      "logps/rejected": -33.555274963378906,
      "loss": 0.9993,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0004924900131300092,
      "rewards/margins": 0.0006568533135578036,
      "rewards/rejected": -0.00016436330042779446,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.0161168575286865,
      "logits/rejected": -2.0146450996398926,
      "logps/chosen": -33.46837615966797,
      "logps/rejected": -32.443458557128906,
      "loss": 1.0002,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.00017420158837921917,
      "rewards/margins": -0.00018708205607254058,
      "rewards/rejected": 0.00036128362989984453,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.102954626083374,
      "logits/rejected": -2.087148666381836,
      "logps/chosen": -34.159751892089844,
      "logps/rejected": -33.093017578125,
      "loss": 1.0001,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.00019638515368569642,
      "rewards/margins": -5.3783714974997565e-05,
      "rewards/rejected": 0.0002501688722986728,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.974347710609436,
      "logits/rejected": -1.9733997583389282,
      "logps/chosen": -33.24498748779297,
      "logps/rejected": -32.44942855834961,
      "loss": 0.9998,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.00037107508978806436,
      "rewards/margins": 0.0002138734853360802,
      "rewards/rejected": 0.00015720158990006894,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9305756092071533,
      "logits/rejected": -1.9409431219100952,
      "logps/chosen": -32.20996856689453,
      "logps/rejected": -35.26984405517578,
      "loss": 1.0002,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -5.276153751765378e-05,
      "rewards/margins": -0.00024183676578104496,
      "rewards/rejected": 0.00018907521734945476,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.0696487426757812,
      "logits/rejected": -2.0630972385406494,
      "logps/chosen": -33.643394470214844,
      "logps/rejected": -29.242889404296875,
      "loss": 0.9997,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 8.579110726714134e-05,
      "rewards/margins": 0.0002504353760741651,
      "rewards/rejected": -0.00016464429791085422,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.9290872812271118,
      "logits/rejected": -1.9312543869018555,
      "logps/chosen": -34.22844696044922,
      "logps/rejected": -30.902618408203125,
      "loss": 0.9996,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.00010604139242786914,
      "rewards/margins": 0.0004059396160300821,
      "rewards/rejected": -0.00029989826725795865,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.9998476387618425,
      "train_runtime": 3251.6099,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}