{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.866639494895935,
      "logits/rejected": -1.870951533317566,
      "logps/chosen": -36.991004943847656,
      "logps/rejected": -33.64551544189453,
      "loss": 0.4968,
      "rewards/accuracies": 0.5694444179534912,
      "rewards/chosen": 0.009355764836072922,
      "rewards/margins": 0.018620232120156288,
      "rewards/rejected": -0.009264465421438217,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9975944757461548,
      "logits/rejected": -2.000239133834839,
      "logps/chosen": -29.655420303344727,
      "logps/rejected": -29.04937171936035,
      "loss": 0.5034,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.007936288602650166,
      "rewards/margins": -0.016377601772546768,
      "rewards/rejected": 0.008441311307251453,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.920345664024353,
      "logits/rejected": -1.917649507522583,
      "logps/chosen": -31.40158462524414,
      "logps/rejected": -33.22117614746094,
      "loss": 0.499,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008706120774149895,
      "rewards/margins": 0.010335424914956093,
      "rewards/rejected": -0.0016293060034513474,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.016953706741333,
      "logits/rejected": -2.008241891860962,
      "logps/chosen": -32.55894088745117,
      "logps/rejected": -32.48887634277344,
      "loss": 0.501,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.010718915611505508,
      "rewards/margins": -0.0035588580649346113,
      "rewards/rejected": 0.014277773909270763,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8621622323989868,
      "logits/rejected": -1.851413369178772,
      "logps/chosen": -33.5220832824707,
      "logps/rejected": -35.4012565612793,
      "loss": 0.5034,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.021240826696157455,
      "rewards/margins": -0.009411826729774475,
      "rewards/rejected": 0.03065265715122223,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9412288665771484,
      "logits/rejected": -1.9431712627410889,
      "logps/chosen": -32.506317138671875,
      "logps/rejected": -33.202842712402344,
      "loss": 0.4846,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05631808191537857,
      "rewards/margins": 0.07519452273845673,
      "rewards/rejected": -0.018876442685723305,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.0726475715637207,
      "logits/rejected": -2.077623128890991,
      "logps/chosen": -33.90425491333008,
      "logps/rejected": -36.56460189819336,
      "loss": 0.4903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05054063722491264,
      "rewards/margins": 0.04797897860407829,
      "rewards/rejected": 0.002561659086495638,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9338359832763672,
      "logits/rejected": -1.9369510412216187,
      "logps/chosen": -34.21551513671875,
      "logps/rejected": -34.527488708496094,
      "loss": 0.4797,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.1273767203092575,
      "rewards/margins": 0.08842838555574417,
      "rewards/rejected": 0.038948334753513336,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9420888423919678,
      "logits/rejected": -1.9465935230255127,
      "logps/chosen": -32.2774772644043,
      "logps/rejected": -32.245147705078125,
      "loss": 0.4887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11396588385105133,
      "rewards/margins": 0.04570900648832321,
      "rewards/rejected": 0.06825686991214752,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.040524482727051,
      "logits/rejected": -2.0385289192199707,
      "logps/chosen": -32.03118133544922,
      "logps/rejected": -31.205692291259766,
      "loss": 0.4799,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.12579473853111267,
      "rewards/margins": 0.08948288857936859,
      "rewards/rejected": 0.036311857402324677,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.234907865524292,
      "eval_logits/rejected": -2.2300779819488525,
      "eval_logps/chosen": -33.90293884277344,
      "eval_logps/rejected": -37.394290924072266,
      "eval_loss": 0.4987158477306366,
      "eval_rewards/accuracies": 0.5103820562362671,
      "eval_rewards/chosen": 0.0789671465754509,
      "eval_rewards/margins": 0.005569713190197945,
      "eval_rewards/rejected": 0.0733974277973175,
      "eval_runtime": 145.9867,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9966247081756592,
      "logits/rejected": -1.9942585229873657,
      "logps/chosen": -33.005943298339844,
      "logps/rejected": -33.908599853515625,
      "loss": 0.4799,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.14272192120552063,
      "rewards/margins": 0.07025818526744843,
      "rewards/rejected": 0.07246376574039459,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.0064611434936523,
      "logits/rejected": -1.9981132745742798,
      "logps/chosen": -32.16605758666992,
      "logps/rejected": -31.968135833740234,
      "loss": 0.4883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16730833053588867,
      "rewards/margins": 0.0512862391769886,
      "rewards/rejected": 0.11602209508419037,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0353620052337646,
      "logits/rejected": -2.0273985862731934,
      "logps/chosen": -30.145355224609375,
      "logps/rejected": -31.892078399658203,
      "loss": 0.477,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.19821622967720032,
      "rewards/margins": 0.10165951400995255,
      "rewards/rejected": 0.09655670821666718,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.965121865272522,
      "logits/rejected": -1.9753497838974,
      "logps/chosen": -31.059722900390625,
      "logps/rejected": -32.391807556152344,
      "loss": 0.4695,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.21596479415893555,
      "rewards/margins": 0.13012099266052246,
      "rewards/rejected": 0.08584379404783249,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8772761821746826,
      "logits/rejected": -1.8784434795379639,
      "logps/chosen": -33.682029724121094,
      "logps/rejected": -34.56599044799805,
      "loss": 0.4566,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.30639737844467163,
      "rewards/margins": 0.19163966178894043,
      "rewards/rejected": 0.11475770175457001,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9283088445663452,
      "logits/rejected": -1.9249660968780518,
      "logps/chosen": -35.78284454345703,
      "logps/rejected": -32.46989059448242,
      "loss": 0.4802,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.23330166935920715,
      "rewards/margins": 0.08158595860004425,
      "rewards/rejected": 0.1517156958580017,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0310397148132324,
      "logits/rejected": -2.0237185955047607,
      "logps/chosen": -33.21892547607422,
      "logps/rejected": -31.1767520904541,
      "loss": 0.443,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.347359299659729,
      "rewards/margins": 0.24229975044727325,
      "rewards/rejected": 0.10505956411361694,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.037632703781128,
      "logits/rejected": -2.0428080558776855,
      "logps/chosen": -31.97260093688965,
      "logps/rejected": -32.14643096923828,
      "loss": 0.4662,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.34331104159355164,
      "rewards/margins": 0.1400861144065857,
      "rewards/rejected": 0.20322492718696594,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0373456478118896,
      "logits/rejected": -2.0346312522888184,
      "logps/chosen": -31.077922821044922,
      "logps/rejected": -31.089244842529297,
      "loss": 0.4725,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.24661760032176971,
      "rewards/margins": 0.10923590511083603,
      "rewards/rejected": 0.13738170266151428,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9075931310653687,
      "logits/rejected": -1.9122419357299805,
      "logps/chosen": -31.088512420654297,
      "logps/rejected": -32.591243743896484,
      "loss": 0.4548,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.3106623888015747,
      "rewards/margins": 0.190850168466568,
      "rewards/rejected": 0.11981221288442612,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.233458995819092,
      "eval_logits/rejected": -2.2286527156829834,
      "eval_logps/chosen": -33.769569396972656,
      "eval_logps/rejected": -37.28458786010742,
      "eval_loss": 0.4955619275569916,
      "eval_rewards/accuracies": 0.5340532064437866,
      "eval_rewards/chosen": 0.15899108350276947,
      "eval_rewards/margins": 0.019772635772824287,
      "eval_rewards/rejected": 0.13921843469142914,
      "eval_runtime": 145.6335,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.020599365234375,
      "logits/rejected": -2.0312187671661377,
      "logps/chosen": -31.51775550842285,
      "logps/rejected": -33.68301773071289,
      "loss": 0.462,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2721289098262787,
      "rewards/margins": 0.16431835293769836,
      "rewards/rejected": 0.10781057924032211,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9131078720092773,
      "logits/rejected": -1.9278091192245483,
      "logps/chosen": -29.563827514648438,
      "logps/rejected": -31.399028778076172,
      "loss": 0.4463,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.3286263942718506,
      "rewards/margins": 0.23313641548156738,
      "rewards/rejected": 0.09548995643854141,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.969555139541626,
      "logits/rejected": -1.9735543727874756,
      "logps/chosen": -32.82552719116211,
      "logps/rejected": -31.39508628845215,
      "loss": 0.4396,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.3551639914512634,
      "rewards/margins": 0.26263612508773804,
      "rewards/rejected": 0.0925278514623642,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9689340591430664,
      "logits/rejected": -1.947165846824646,
      "logps/chosen": -33.572689056396484,
      "logps/rejected": -34.84184265136719,
      "loss": 0.4355,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3486354947090149,
      "rewards/margins": 0.2832663357257843,
      "rewards/rejected": 0.0653691440820694,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0102758407592773,
      "logits/rejected": -2.006990909576416,
      "logps/chosen": -32.49989700317383,
      "logps/rejected": -35.99126434326172,
      "loss": 0.472,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.2551037669181824,
      "rewards/margins": 0.11490776389837265,
      "rewards/rejected": 0.1401960253715515,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.877345323562622,
      "logits/rejected": -1.8749182224273682,
      "logps/chosen": -33.7043571472168,
      "logps/rejected": -35.259422302246094,
      "loss": 0.4671,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.2904246747493744,
      "rewards/margins": 0.14330394566059113,
      "rewards/rejected": 0.14712072908878326,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8626378774642944,
      "logits/rejected": -1.8601328134536743,
      "logps/chosen": -33.90363311767578,
      "logps/rejected": -31.588598251342773,
      "loss": 0.457,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.29638105630874634,
      "rewards/margins": 0.1867421716451645,
      "rewards/rejected": 0.10963888466358185,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.965528130531311,
      "logits/rejected": -1.9551260471343994,
      "logps/chosen": -34.74420166015625,
      "logps/rejected": -31.622207641601562,
      "loss": 0.4442,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.3674633204936981,
      "rewards/margins": 0.23619195818901062,
      "rewards/rejected": 0.1312713772058487,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0614726543426514,
      "logits/rejected": -2.0466229915618896,
      "logps/chosen": -30.37356948852539,
      "logps/rejected": -32.3351936340332,
      "loss": 0.4671,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.33033472299575806,
      "rewards/margins": 0.14034779369831085,
      "rewards/rejected": 0.1899869292974472,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9327377080917358,
      "logits/rejected": -1.9302759170532227,
      "logps/chosen": -32.0928955078125,
      "logps/rejected": -30.652236938476562,
      "loss": 0.41,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.5005279779434204,
      "rewards/margins": 0.4017825126647949,
      "rewards/rejected": 0.09874548017978668,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2308623790740967,
      "eval_logits/rejected": -2.226059913635254,
      "eval_logps/chosen": -33.76142501831055,
      "eval_logps/rejected": -37.284828186035156,
      "eval_loss": 0.4937135577201843,
      "eval_rewards/accuracies": 0.5361295938491821,
      "eval_rewards/chosen": 0.16387464106082916,
      "eval_rewards/margins": 0.02479831501841545,
      "eval_rewards/rejected": 0.1390763521194458,
      "eval_runtime": 145.6766,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.916757345199585,
      "logits/rejected": -1.9136326313018799,
      "logps/chosen": -31.0240421295166,
      "logps/rejected": -33.553138732910156,
      "loss": 0.4465,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3475847542285919,
      "rewards/margins": 0.23417536914348602,
      "rewards/rejected": 0.1134093850851059,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.9669981002807617,
      "logits/rejected": -1.9548496007919312,
      "logps/chosen": -34.02072525024414,
      "logps/rejected": -33.40859603881836,
      "loss": 0.4379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3452104926109314,
      "rewards/margins": 0.26706355810165405,
      "rewards/rejected": 0.07814691960811615,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -2.0023112297058105,
      "logits/rejected": -2.0009615421295166,
      "logps/chosen": -32.86361312866211,
      "logps/rejected": -32.265541076660156,
      "loss": 0.4439,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.37330904603004456,
      "rewards/margins": 0.24488253891468048,
      "rewards/rejected": 0.12842652201652527,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.0890090465545654,
      "logits/rejected": -2.073373317718506,
      "logps/chosen": -33.47917556762695,
      "logps/rejected": -32.80946350097656,
      "loss": 0.4446,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.42013058066368103,
      "rewards/margins": 0.23498812317848206,
      "rewards/rejected": 0.18514248728752136,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.9615952968597412,
      "logits/rejected": -1.9607601165771484,
      "logps/chosen": -32.57384490966797,
      "logps/rejected": -32.25366973876953,
      "loss": 0.4326,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.4249538779258728,
      "rewards/margins": 0.2980697453022003,
      "rewards/rejected": 0.12688413262367249,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9177497625350952,
      "logits/rejected": -1.9280321598052979,
      "logps/chosen": -31.600200653076172,
      "logps/rejected": -35.003135681152344,
      "loss": 0.4555,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.3626941442489624,
      "rewards/margins": 0.19132137298583984,
      "rewards/rejected": 0.17137278616428375,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.0559232234954834,
      "logits/rejected": -2.0493996143341064,
      "logps/chosen": -33.01537322998047,
      "logps/rejected": -28.992477416992188,
      "loss": 0.4418,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.3819587230682373,
      "rewards/margins": 0.2415906935930252,
      "rewards/rejected": 0.14036807417869568,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9163579940795898,
      "logits/rejected": -1.9185268878936768,
      "logps/chosen": -33.650962829589844,
      "logps/rejected": -30.753406524658203,
      "loss": 0.4369,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.3528522849082947,
      "rewards/margins": 0.28131863474845886,
      "rewards/rejected": 0.07153360545635223,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.46439293204963983,
      "train_runtime": 3253.2697,
      "train_samples_per_second": 0.946,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}