{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.866548776626587,
      "logits/rejected": -1.870866298675537,
      "logps/chosen": -36.98152160644531,
      "logps/rejected": -33.65525436401367,
      "loss": 0.6907,
      "rewards/accuracies": 0.5833333134651184,
      "rewards/chosen": 0.0025075911544263363,
      "rewards/margins": 0.005025296472012997,
      "rewards/rejected": -0.002517705550417304,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9975509643554688,
      "logits/rejected": -2.000202178955078,
      "logps/chosen": -29.649311065673828,
      "logps/rejected": -29.060571670532227,
      "loss": 0.6937,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.000711599481292069,
      "rewards/margins": -0.0009990204125642776,
      "rewards/rejected": 0.00028742075664922595,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9204410314559937,
      "logits/rejected": -1.9177614450454712,
      "logps/chosen": -31.389001846313477,
      "logps/rejected": -33.2293701171875,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0027091833762824535,
      "rewards/margins": 0.0038000899367034435,
      "rewards/rejected": -0.0010909067932516336,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0175061225891113,
      "logits/rejected": -2.0087640285491943,
      "logps/chosen": -32.5477180480957,
      "logps/rejected": -32.5131950378418,
      "loss": 0.6917,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.002909052651375532,
      "rewards/margins": 0.0029616341926157475,
      "rewards/rejected": -5.258135570329614e-05,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.862919569015503,
      "logits/rejected": -1.8521493673324585,
      "logps/chosen": -33.53850555419922,
      "logps/rejected": -35.44303512573242,
      "loss": 0.6927,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.001897345995530486,
      "rewards/margins": 0.0009661337244324386,
      "rewards/rejected": 0.0009312118636444211,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.941716194152832,
      "logits/rejected": -1.943652868270874,
      "logps/chosen": -32.55483627319336,
      "logps/rejected": -33.212310791015625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004534821957349777,
      "rewards/margins": 0.008627313189208508,
      "rewards/rejected": -0.004092490766197443,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.072801113128662,
      "logits/rejected": -2.0777623653411865,
      "logps/chosen": -33.98583221435547,
      "logps/rejected": -36.61463165283203,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0002649508533068001,
      "rewards/margins": 0.0048407576978206635,
      "rewards/rejected": -0.004575806204229593,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.933315634727478,
      "logits/rejected": -1.9364687204360962,
      "logps/chosen": -34.30742645263672,
      "logps/rejected": -34.65543746948242,
      "loss": 0.6843,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01203834917396307,
      "rewards/margins": 0.0183414313942194,
      "rewards/rejected": -0.006303082220256329,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9412227869033813,
      "logits/rejected": -1.9457390308380127,
      "logps/chosen": -32.38041305541992,
      "logps/rejected": -32.36061096191406,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008700890466570854,
      "rewards/margins": 0.0088711092248559,
      "rewards/rejected": -0.0001702193112578243,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.038828134536743,
      "logits/rejected": -2.0368473529815674,
      "logps/chosen": -32.14335250854492,
      "logps/rejected": -31.2996768951416,
      "loss": 0.6868,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.009748170152306557,
      "rewards/margins": 0.013094509951770306,
      "rewards/rejected": -0.0033463402651250362,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2332358360290527,
      "eval_logits/rejected": -2.228397846221924,
      "eval_logps/chosen": -34.052730560302734,
      "eval_logps/rejected": -37.55271911621094,
      "eval_loss": 0.692415714263916,
      "eval_rewards/accuracies": 0.5282392501831055,
      "eval_rewards/chosen": -0.0018174793804064393,
      "eval_rewards/margins": 0.0017920632380992174,
      "eval_rewards/rejected": -0.0036095422692596912,
      "eval_runtime": 145.8681,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9926027059555054,
      "logits/rejected": -1.9902327060699463,
      "logps/chosen": -33.1365852355957,
      "logps/rejected": -34.04566192626953,
      "loss": 0.6877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010723221115767956,
      "rewards/margins": 0.012352446094155312,
      "rewards/rejected": -0.0016292240470647812,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.0035624504089355,
      "logits/rejected": -1.995218276977539,
      "logps/chosen": -32.34514617919922,
      "logps/rejected": -32.111610412597656,
      "loss": 0.6909,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.009975815191864967,
      "rewards/margins": 0.004986769054085016,
      "rewards/rejected": 0.004989045672118664,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0313966274261475,
      "logits/rejected": -2.0234367847442627,
      "logps/chosen": -30.32912254333496,
      "logps/rejected": -32.07999801635742,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014659431762993336,
      "rewards/margins": 0.017358481884002686,
      "rewards/rejected": -0.0026990489568561316,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9624744653701782,
      "logits/rejected": -1.9726974964141846,
      "logps/chosen": -31.2264347076416,
      "logps/rejected": -32.56539535522461,
      "loss": 0.6823,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.019322697073221207,
      "rewards/margins": 0.02237415872514248,
      "rewards/rejected": -0.0030514628160744905,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8730823993682861,
      "logits/rejected": -1.8742510080337524,
      "logps/chosen": -33.906436920166016,
      "logps/rejected": -34.79541778564453,
      "loss": 0.6777,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.028625473380088806,
      "rewards/margins": 0.03244180604815483,
      "rewards/rejected": -0.0038163296412676573,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.924594521522522,
      "logits/rejected": -1.9211914539337158,
      "logps/chosen": -35.99129867553711,
      "logps/rejected": -32.720947265625,
      "loss": 0.6845,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.0180378220975399,
      "rewards/margins": 0.01785714365541935,
      "rewards/rejected": 0.0001806807704269886,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.025442600250244,
      "logits/rejected": -2.0181422233581543,
      "logps/chosen": -33.491790771484375,
      "logps/rejected": -31.42473793029785,
      "loss": 0.6749,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.030607115477323532,
      "rewards/margins": 0.03789608180522919,
      "rewards/rejected": -0.007288967724889517,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0314249992370605,
      "logits/rejected": -2.0366628170013428,
      "logps/chosen": -32.224815368652344,
      "logps/rejected": -32.438621520996094,
      "loss": 0.68,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03199741616845131,
      "rewards/margins": 0.02734515443444252,
      "rewards/rejected": 0.004652261734008789,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.032400131225586,
      "logits/rejected": -2.0296273231506348,
      "logps/chosen": -31.274377822875977,
      "logps/rejected": -31.340478897094727,
      "loss": 0.6818,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.021457459777593613,
      "rewards/margins": 0.023684043437242508,
      "rewards/rejected": -0.002226583892479539,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.902413010597229,
      "logits/rejected": -1.9070699214935303,
      "logps/chosen": -31.3071231842041,
      "logps/rejected": -32.819236755371094,
      "loss": 0.6774,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.029915904626250267,
      "rewards/margins": 0.03274631127715111,
      "rewards/rejected": -0.0028304127044975758,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2295496463775635,
      "eval_logits/rejected": -2.224731206893921,
      "eval_logps/chosen": -34.047569274902344,
      "eval_logps/rejected": -37.568965911865234,
      "eval_loss": 0.6914328336715698,
      "eval_rewards/accuracies": 0.5510797500610352,
      "eval_rewards/chosen": -0.0013017345918342471,
      "eval_rewards/margins": 0.003932718187570572,
      "eval_rewards/rejected": -0.005234452895820141,
      "eval_runtime": 145.6523,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.0154383182525635,
      "logits/rejected": -2.0260632038116455,
      "logps/chosen": -31.753280639648438,
      "logps/rejected": -33.95705032348633,
      "loss": 0.6781,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.021802183240652084,
      "rewards/margins": 0.031237006187438965,
      "rewards/rejected": -0.009434822015464306,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9075886011123657,
      "logits/rejected": -1.9223048686981201,
      "logps/chosen": -29.80865478515625,
      "logps/rejected": -31.63385009765625,
      "loss": 0.675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03028874099254608,
      "rewards/margins": 0.037856362760066986,
      "rewards/rejected": -0.007567620370537043,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9642608165740967,
      "logits/rejected": -1.9682174921035767,
      "logps/chosen": -33.096744537353516,
      "logps/rejected": -31.623376846313477,
      "loss": 0.6746,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.032072193920612335,
      "rewards/margins": 0.03947969526052475,
      "rewards/rejected": -0.0074075027368962765,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.962222695350647,
      "logits/rejected": -1.9404484033584595,
      "logps/chosen": -33.80894088745117,
      "logps/rejected": -35.11548614501953,
      "loss": 0.6688,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.034481119364500046,
      "rewards/margins": 0.050950419157743454,
      "rewards/rejected": -0.016469307243824005,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0041451454162598,
      "logits/rejected": -2.0008480548858643,
      "logps/chosen": -32.71742630004883,
      "logps/rejected": -36.27611541748047,
      "loss": 0.6808,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.020764300599694252,
      "rewards/margins": 0.02588369883596897,
      "rewards/rejected": -0.005119399167597294,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.871769666671753,
      "logits/rejected": -1.8693596124649048,
      "logps/chosen": -33.992244720458984,
      "logps/rejected": -35.55255126953125,
      "loss": 0.6816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.019615134224295616,
      "rewards/margins": 0.024408066645264626,
      "rewards/rejected": -0.004792929161339998,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8571555614471436,
      "logits/rejected": -1.8547788858413696,
      "logps/chosen": -34.180908203125,
      "logps/rejected": -31.8300838470459,
      "loss": 0.6802,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.021669462323188782,
      "rewards/margins": 0.027544772252440453,
      "rewards/rejected": -0.00587531179189682,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9601259231567383,
      "logits/rejected": -1.949663519859314,
      "logps/chosen": -34.989627838134766,
      "logps/rejected": -31.897205352783203,
      "loss": 0.6728,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03670142963528633,
      "rewards/margins": 0.042322538793087006,
      "rewards/rejected": -0.005621104035526514,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.054929494857788,
      "logits/rejected": -2.040048122406006,
      "logps/chosen": -30.71946144104004,
      "logps/rejected": -32.615081787109375,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020466715097427368,
      "rewards/margins": 0.016791274771094322,
      "rewards/rejected": 0.0036754377651959658,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9249824285507202,
      "logits/rejected": -1.9225151538848877,
      "logps/chosen": -32.352317810058594,
      "logps/rejected": -30.916645050048828,
      "loss": 0.6615,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.05747910216450691,
      "rewards/margins": 0.06746286153793335,
      "rewards/rejected": -0.009983762167394161,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2262814044952393,
      "eval_logits/rejected": -2.221453905105591,
      "eval_logps/chosen": -34.07697677612305,
      "eval_logps/rejected": -37.600772857666016,
      "eval_loss": 0.6913371086120605,
      "eval_rewards/accuracies": 0.5477575063705444,
      "eval_rewards/chosen": -0.004243030212819576,
      "eval_rewards/margins": 0.004172220826148987,
      "eval_rewards/rejected": -0.008415251038968563,
      "eval_runtime": 145.7082,
      "eval_samples_per_second": 2.354,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.9102771282196045,
      "logits/rejected": -1.9070390462875366,
      "logps/chosen": -31.315908432006836,
      "logps/rejected": -33.82908248901367,
      "loss": 0.6755,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.02874402143061161,
      "rewards/margins": 0.037437256425619125,
      "rewards/rejected": -0.008693234995007515,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.960473656654358,
      "logits/rejected": -1.9483349323272705,
      "logps/chosen": -34.295162200927734,
      "logps/rejected": -33.6617431640625,
      "loss": 0.6729,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03009115532040596,
      "rewards/margins": 0.04238158464431763,
      "rewards/rejected": -0.012290433049201965,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -1.9958908557891846,
      "logits/rejected": -1.994442343711853,
      "logps/chosen": -33.12481689453125,
      "logps/rejected": -32.55192184448242,
      "loss": 0.6726,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.036097489297389984,
      "rewards/margins": 0.04333154112100601,
      "rewards/rejected": -0.007234054151922464,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.0822036266326904,
      "logits/rejected": -2.06657075881958,
      "logps/chosen": -33.78117752075195,
      "logps/rejected": -33.087318420410156,
      "loss": 0.6755,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03982168063521385,
      "rewards/margins": 0.036749765276908875,
      "rewards/rejected": 0.0030719186179339886,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.9546077251434326,
      "logits/rejected": -1.9537696838378906,
      "logps/chosen": -32.80642318725586,
      "logps/rejected": -32.51500701904297,
      "loss": 0.6684,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04756731912493706,
      "rewards/margins": 0.05255315452814102,
      "rewards/rejected": -0.004985834006220102,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9106451272964478,
      "logits/rejected": -1.9209150075912476,
      "logps/chosen": -31.8604736328125,
      "logps/rejected": -35.30449676513672,
      "loss": 0.676,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.034421421587467194,
      "rewards/margins": 0.03599538281559944,
      "rewards/rejected": -0.0015739675145596266,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.051037311553955,
      "logits/rejected": -2.044602870941162,
      "logps/chosen": -33.32126235961914,
      "logps/rejected": -29.267892837524414,
      "loss": 0.6752,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.033070918172597885,
      "rewards/margins": 0.03721793740987778,
      "rewards/rejected": -0.004147022031247616,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9098504781723022,
      "logits/rejected": -1.9120371341705322,
      "logps/chosen": -33.857635498046875,
      "logps/rejected": -30.970882415771484,
      "loss": 0.6702,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.03814157843589783,
      "rewards/margins": 0.04796718806028366,
      "rewards/rejected": -0.009825612418353558,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.6806787961489195,
      "train_runtime": 3251.1509,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}