{
  "best_metric": 0.1529119908809662,
  "best_model_checkpoint": "saves/Llama-3.1-8B-Instruct/lora/saa-800/checkpoint-450",
  "epoch": 10.0,
  "eval_steps": 50,
  "global_step": 450,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 6.713047504425049,
      "learning_rate": 1.111111111111111e-06,
      "logits/chosen": -0.4419662356376648,
      "logits/rejected": -0.5169282555580139,
      "logps/chosen": -1.7502187490463257,
      "logps/rejected": -2.110954523086548,
      "loss": 1.8058,
      "odds_ratio_loss": 15.742452621459961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17502185702323914,
      "rewards/margins": 0.03607357665896416,
      "rewards/rejected": -0.21109545230865479,
      "sft_loss": 0.23150739073753357,
      "step": 10
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 7.350757122039795,
      "learning_rate": 2.222222222222222e-06,
      "logits/chosen": -0.390705406665802,
      "logits/rejected": -0.4706195294857025,
      "logps/chosen": -1.6888424158096313,
      "logps/rejected": -2.1693763732910156,
      "loss": 1.7416,
      "odds_ratio_loss": 15.1643705368042,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.16888423264026642,
      "rewards/margins": 0.048053398728370667,
      "rewards/rejected": -0.21693763136863708,
      "sft_loss": 0.22513218224048615,
      "step": 20
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 8.413761138916016,
      "learning_rate": 3.3333333333333333e-06,
      "logits/chosen": -0.42878788709640503,
      "logits/rejected": -0.5077282190322876,
      "logps/chosen": -1.698947548866272,
      "logps/rejected": -2.1924192905426025,
      "loss": 1.7505,
      "odds_ratio_loss": 15.268714904785156,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1698947548866272,
      "rewards/margins": 0.04934718459844589,
      "rewards/rejected": -0.2192419320344925,
      "sft_loss": 0.22367195785045624,
      "step": 30
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 6.453053951263428,
      "learning_rate": 4.444444444444444e-06,
      "logits/chosen": -0.40729817748069763,
      "logits/rejected": -0.480067640542984,
      "logps/chosen": -1.677170753479004,
      "logps/rejected": -1.9914417266845703,
      "loss": 1.736,
      "odds_ratio_loss": 15.256231307983398,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1677170693874359,
      "rewards/margins": 0.03142711520195007,
      "rewards/rejected": -0.1991441696882248,
      "sft_loss": 0.21038159728050232,
      "step": 40
    },
    {
      "epoch": 1.1111111111111112,
      "grad_norm": 5.584766864776611,
      "learning_rate": 4.998119881260576e-06,
      "logits/chosen": -0.4123212695121765,
      "logits/rejected": -0.49063047766685486,
      "logps/chosen": -1.4437320232391357,
      "logps/rejected": -1.951922059059143,
      "loss": 1.4945,
      "odds_ratio_loss": 13.130969047546387,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.14437320828437805,
      "rewards/margins": 0.050819020718336105,
      "rewards/rejected": -0.19519223272800446,
      "sft_loss": 0.18137237429618835,
      "step": 50
    },
    {
      "epoch": 1.1111111111111112,
      "eval_logits/chosen": -0.3793938159942627,
      "eval_logits/rejected": -0.45482414960861206,
      "eval_logps/chosen": -1.228076696395874,
      "eval_logps/rejected": -1.7094627618789673,
      "eval_loss": 1.276250958442688,
      "eval_odds_ratio_loss": 11.260908126831055,
      "eval_rewards/accuracies": 0.7875000238418579,
      "eval_rewards/chosen": -0.12280768156051636,
      "eval_rewards/margins": 0.04813859239220619,
      "eval_rewards/rejected": -0.17094627022743225,
      "eval_runtime": 3.3465,
      "eval_samples_per_second": 23.905,
      "eval_sft_loss": 0.15016020834445953,
      "eval_steps_per_second": 11.953,
      "step": 50
    },
    {
      "epoch": 1.3333333333333333,
      "grad_norm": 3.5215771198272705,
      "learning_rate": 4.983095894354858e-06,
      "logits/chosen": -0.4166947901248932,
      "logits/rejected": -0.49852150678634644,
      "logps/chosen": -1.0756200551986694,
      "logps/rejected": -1.5570634603500366,
      "loss": 1.1242,
      "odds_ratio_loss": 9.97706127166748,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.10756200551986694,
      "rewards/margins": 0.04814432933926582,
      "rewards/rejected": -0.15570633113384247,
      "sft_loss": 0.1265355795621872,
      "step": 60
    },
    {
      "epoch": 1.5555555555555556,
      "grad_norm": 3.005864381790161,
      "learning_rate": 4.953138276568462e-06,
      "logits/chosen": -0.45257633924484253,
      "logits/rejected": -0.5196300148963928,
      "logps/chosen": -0.6806862950325012,
      "logps/rejected": -1.1254621744155884,
      "loss": 0.7274,
      "odds_ratio_loss": 6.491485595703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06806863099336624,
      "rewards/margins": 0.044477589428424835,
      "rewards/rejected": -0.11254620552062988,
      "sft_loss": 0.07826091349124908,
      "step": 70
    },
    {
      "epoch": 1.7777777777777777,
      "grad_norm": 3.241006851196289,
      "learning_rate": 4.908427196539701e-06,
      "logits/chosen": -0.4250833988189697,
      "logits/rejected": -0.49341732263565063,
      "logps/chosen": -0.586665153503418,
      "logps/rejected": -0.9428589940071106,
      "loss": 0.6374,
      "odds_ratio_loss": 5.749706268310547,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.05866651609539986,
      "rewards/margins": 0.03561938554048538,
      "rewards/rejected": -0.09428589791059494,
      "sft_loss": 0.06239296868443489,
      "step": 80
    },
    {
      "epoch": 2.0,
      "grad_norm": 3.3662965297698975,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -0.39320772886276245,
      "logits/rejected": -0.48544201254844666,
      "logps/chosen": -0.3357451558113098,
      "logps/rejected": -0.9199529886245728,
      "loss": 0.3774,
      "odds_ratio_loss": 3.414803981781006,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.03357451409101486,
      "rewards/margins": 0.05842079594731331,
      "rewards/rejected": -0.09199531376361847,
      "sft_loss": 0.03593815118074417,
      "step": 90
    },
    {
      "epoch": 2.2222222222222223,
      "grad_norm": 1.9536067247390747,
      "learning_rate": 4.775907352415367e-06,
      "logits/chosen": -0.3746485710144043,
      "logits/rejected": -0.46515822410583496,
      "logps/chosen": -0.22609436511993408,
      "logps/rejected": -0.7698042392730713,
      "loss": 0.2666,
      "odds_ratio_loss": 2.4158308506011963,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.022609438747167587,
      "rewards/margins": 0.05437099188566208,
      "rewards/rejected": -0.07698042690753937,
      "sft_loss": 0.02498885616660118,
      "step": 100
    },
    {
      "epoch": 2.2222222222222223,
      "eval_logits/chosen": -0.36658304929733276,
      "eval_logits/rejected": -0.4319043755531311,
      "eval_logps/chosen": -0.2119404524564743,
      "eval_logps/rejected": -0.7287805676460266,
      "eval_loss": 0.24907442927360535,
      "eval_odds_ratio_loss": 2.2483649253845215,
      "eval_rewards/accuracies": 0.824999988079071,
      "eval_rewards/chosen": -0.02119404636323452,
      "eval_rewards/margins": 0.05168401449918747,
      "eval_rewards/rejected": -0.07287804782390594,
      "eval_runtime": 3.3336,
      "eval_samples_per_second": 23.998,
      "eval_sft_loss": 0.02423796057701111,
      "eval_steps_per_second": 11.999,
      "step": 100
    },
    {
      "epoch": 2.4444444444444446,
      "grad_norm": 1.7341715097427368,
      "learning_rate": 4.688895578255228e-06,
      "logits/chosen": -0.38399261236190796,
      "logits/rejected": -0.45104750990867615,
      "logps/chosen": -0.1595337688922882,
      "logps/rejected": -0.6476808190345764,
      "loss": 0.1984,
      "odds_ratio_loss": 1.7913240194320679,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.01595337688922882,
      "rewards/margins": 0.048814695328474045,
      "rewards/rejected": -0.06476806849241257,
      "sft_loss": 0.0192393995821476,
      "step": 110
    },
    {
      "epoch": 2.6666666666666665,
      "grad_norm": 1.6577801704406738,
      "learning_rate": 4.588719528532342e-06,
      "logits/chosen": -0.42996135354042053,
      "logits/rejected": -0.5020943880081177,
      "logps/chosen": -0.11305193603038788,
      "logps/rejected": -0.6292703747749329,
      "loss": 0.1528,
      "odds_ratio_loss": 1.378528356552124,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.011305193416774273,
      "rewards/margins": 0.05162184312939644,
      "rewards/rejected": -0.06292703002691269,
      "sft_loss": 0.014913685619831085,
      "step": 120
    },
    {
      "epoch": 2.888888888888889,
      "grad_norm": 0.804415762424469,
      "learning_rate": 4.475981673796899e-06,
      "logits/chosen": -0.4010156989097595,
      "logits/rejected": -0.4532565474510193,
      "logps/chosen": -0.11344721168279648,
      "logps/rejected": -0.5929852724075317,
      "loss": 0.1557,
      "odds_ratio_loss": 1.430558204650879,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.011344721540808678,
      "rewards/margins": 0.047953806817531586,
      "rewards/rejected": -0.059298526495695114,
      "sft_loss": 0.012685336172580719,
      "step": 130
    },
    {
      "epoch": 3.111111111111111,
      "grad_norm": 1.188767910003662,
      "learning_rate": 4.351360032772512e-06,
      "logits/chosen": -0.37222054600715637,
      "logits/rejected": -0.4317198395729065,
      "logps/chosen": -0.10829365253448486,
      "logps/rejected": -0.6532029509544373,
      "loss": 0.1518,
      "odds_ratio_loss": 1.385451078414917,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.010829364880919456,
      "rewards/margins": 0.05449093505740166,
      "rewards/rejected": -0.06532029807567596,
      "sft_loss": 0.013293206691741943,
      "step": 140
    },
    {
      "epoch": 3.3333333333333335,
      "grad_norm": 1.1802352666854858,
      "learning_rate": 4.215604094671835e-06,
      "logits/chosen": -0.3585512936115265,
      "logits/rejected": -0.4357479512691498,
      "logps/chosen": -0.07403306663036346,
      "logps/rejected": -0.7158663868904114,
      "loss": 0.1014,
      "odds_ratio_loss": 0.9041417837142944,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007403307594358921,
      "rewards/margins": 0.06418333202600479,
      "rewards/rejected": -0.07158664613962173,
      "sft_loss": 0.010936584323644638,
      "step": 150
    },
    {
      "epoch": 3.3333333333333335,
      "eval_logits/chosen": -0.3283754289150238,
      "eval_logits/rejected": -0.3820287585258484,
      "eval_logps/chosen": -0.1292387694120407,
      "eval_logps/rejected": -0.6057685017585754,
      "eval_loss": 0.16315264999866486,
      "eval_odds_ratio_loss": 1.4634513854980469,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.012923876754939556,
      "eval_rewards/margins": 0.047652970999479294,
      "eval_rewards/rejected": -0.060576848685741425,
      "eval_runtime": 3.3355,
      "eval_samples_per_second": 23.984,
      "eval_sft_loss": 0.016807515174150467,
      "eval_steps_per_second": 11.992,
      "step": 150
    },
    {
      "epoch": 3.5555555555555554,
      "grad_norm": 2.5364763736724854,
      "learning_rate": 4.069530311680247e-06,
      "logits/chosen": -0.37899985909461975,
      "logits/rejected": -0.4429406225681305,
      "logps/chosen": -0.09646956622600555,
      "logps/rejected": -0.5016245245933533,
      "loss": 0.1441,
      "odds_ratio_loss": 1.3272716999053955,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.00964695680886507,
      "rewards/margins": 0.04051550105214119,
      "rewards/rejected": -0.050162456929683685,
      "sft_loss": 0.011343193240463734,
      "step": 160
    },
    {
      "epoch": 3.7777777777777777,
      "grad_norm": 1.1225122213363647,
      "learning_rate": 3.914017188716347e-06,
      "logits/chosen": -0.32853808999061584,
      "logits/rejected": -0.3947625160217285,
      "logps/chosen": -0.0880068689584732,
      "logps/rejected": -0.5245143175125122,
      "loss": 0.1251,
      "odds_ratio_loss": 1.1624243259429932,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.00880068726837635,
      "rewards/margins": 0.04365074634552002,
      "rewards/rejected": -0.05245143175125122,
      "sft_loss": 0.00884676817804575,
      "step": 170
    },
    {
      "epoch": 4.0,
      "grad_norm": 2.3566975593566895,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -0.394605427980423,
      "logits/rejected": -0.4463408589363098,
      "logps/chosen": -0.09484803676605225,
      "logps/rejected": -0.5903218984603882,
      "loss": 0.1306,
      "odds_ratio_loss": 1.1676770448684692,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.00948480237275362,
      "rewards/margins": 0.049547381699085236,
      "rewards/rejected": -0.05903219059109688,
      "sft_loss": 0.013791071251034737,
      "step": 180
    },
    {
      "epoch": 4.222222222222222,
      "grad_norm": 1.111855387687683,
      "learning_rate": 3.578465164203134e-06,
      "logits/chosen": -0.3551129698753357,
      "logits/rejected": -0.43426722288131714,
      "logps/chosen": -0.06811969727277756,
      "logps/rejected": -0.6758677959442139,
      "loss": 0.0949,
      "odds_ratio_loss": 0.8656018376350403,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.006811970379203558,
      "rewards/margins": 0.06077481061220169,
      "rewards/rejected": -0.06758677959442139,
      "sft_loss": 0.008338207378983498,
      "step": 190
    },
    {
      "epoch": 4.444444444444445,
      "grad_norm": 1.3708534240722656,
      "learning_rate": 3.400444312011776e-06,
      "logits/chosen": -0.345875084400177,
      "logits/rejected": -0.3952234089374542,
      "logps/chosen": -0.10109534114599228,
      "logps/rejected": -0.5074071288108826,
      "loss": 0.1429,
      "odds_ratio_loss": 1.325073003768921,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.010109534487128258,
      "rewards/margins": 0.04063117876648903,
      "rewards/rejected": -0.05074070766568184,
      "sft_loss": 0.010358814150094986,
      "step": 200
    },
    {
      "epoch": 4.444444444444445,
      "eval_logits/chosen": -0.32978400588035583,
      "eval_logits/rejected": -0.38182884454727173,
      "eval_logps/chosen": -0.121058389544487,
      "eval_logps/rejected": -0.5841361284255981,
      "eval_loss": 0.1533508002758026,
      "eval_odds_ratio_loss": 1.3751581907272339,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.01210583932697773,
      "eval_rewards/margins": 0.04630777984857559,
      "eval_rewards/rejected": -0.058413613587617874,
      "eval_runtime": 3.3378,
      "eval_samples_per_second": 23.968,
      "eval_sft_loss": 0.015834979712963104,
      "eval_steps_per_second": 11.984,
      "step": 200
    },
    {
      "epoch": 4.666666666666667,
      "grad_norm": 1.4620966911315918,
      "learning_rate": 3.217008081777726e-06,
      "logits/chosen": -0.3843366503715515,
      "logits/rejected": -0.4375368058681488,
      "logps/chosen": -0.08334760367870331,
      "logps/rejected": -0.5660797357559204,
      "loss": 0.1171,
      "odds_ratio_loss": 1.0661273002624512,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.00833476148545742,
      "rewards/margins": 0.04827320575714111,
      "rewards/rejected": -0.05660796910524368,
      "sft_loss": 0.010473220609128475,
      "step": 210
    },
    {
      "epoch": 4.888888888888889,
      "grad_norm": 1.2518880367279053,
      "learning_rate": 3.0292596805735275e-06,
      "logits/chosen": -0.3598805367946625,
      "logits/rejected": -0.4199501872062683,
      "logps/chosen": -0.07187429815530777,
      "logps/rejected": -0.5510466694831848,
      "loss": 0.1029,
      "odds_ratio_loss": 0.9251865148544312,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.007187430746853352,
      "rewards/margins": 0.047917239367961884,
      "rewards/rejected": -0.05510466545820236,
      "sft_loss": 0.010374334640800953,
      "step": 220
    },
    {
      "epoch": 5.111111111111111,
      "grad_norm": 1.3117250204086304,
      "learning_rate": 2.8383282493753282e-06,
      "logits/chosen": -0.3635903596878052,
      "logits/rejected": -0.43442487716674805,
      "logps/chosen": -0.06123008579015732,
      "logps/rejected": -0.7030965089797974,
      "loss": 0.0878,
      "odds_ratio_loss": 0.7977678179740906,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.006123009137809277,
      "rewards/margins": 0.06418663263320923,
      "rewards/rejected": -0.07030965387821198,
      "sft_loss": 0.007975312881171703,
      "step": 230
    },
    {
      "epoch": 5.333333333333333,
      "grad_norm": 2.640739679336548,
      "learning_rate": 2.6453620722761897e-06,
      "logits/chosen": -0.350188285112381,
      "logits/rejected": -0.4023573398590088,
      "logps/chosen": -0.08658941090106964,
      "logps/rejected": -0.6677496433258057,
      "loss": 0.1183,
      "odds_ratio_loss": 1.068228006362915,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.00865894090384245,
      "rewards/margins": 0.05811602994799614,
      "rewards/rejected": -0.06677497178316116,
      "sft_loss": 0.011527000926434994,
      "step": 240
    },
    {
      "epoch": 5.555555555555555,
      "grad_norm": 1.3426445722579956,
      "learning_rate": 2.4515216705704396e-06,
      "logits/chosen": -0.3609291911125183,
      "logits/rejected": -0.4190797209739685,
      "logps/chosen": -0.0702720507979393,
      "logps/rejected": -0.596028745174408,
      "loss": 0.1007,
      "odds_ratio_loss": 0.925524115562439,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.00702720507979393,
      "rewards/margins": 0.052575670182704926,
      "rewards/rejected": -0.059602878987789154,
      "sft_loss": 0.008168576285243034,
      "step": 250
    },
    {
      "epoch": 5.555555555555555,
      "eval_logits/chosen": -0.32348352670669556,
      "eval_logits/rejected": -0.37402915954589844,
      "eval_logps/chosen": -0.12062982469797134,
      "eval_logps/rejected": -0.6407424807548523,
      "eval_loss": 0.15295907855033875,
      "eval_odds_ratio_loss": 1.3704458475112915,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.012062981724739075,
      "eval_rewards/margins": 0.05201127007603645,
      "eval_rewards/rejected": -0.06407425552606583,
      "eval_runtime": 3.3317,
      "eval_samples_per_second": 24.012,
      "eval_sft_loss": 0.01591448113322258,
      "eval_steps_per_second": 12.006,
      "step": 250
    },
    {
      "epoch": 5.777777777777778,
      "grad_norm": 1.4032063484191895,
      "learning_rate": 2.2579728232420524e-06,
      "logits/chosen": -0.40077710151672363,
      "logits/rejected": -0.4527947008609772,
      "logps/chosen": -0.0916273444890976,
      "logps/rejected": -0.5904611349105835,
      "loss": 0.127,
      "odds_ratio_loss": 1.1558631658554077,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.009162734262645245,
      "rewards/margins": 0.049883387982845306,
      "rewards/rejected": -0.05904611945152283,
      "sft_loss": 0.011430850252509117,
      "step": 260
    },
    {
      "epoch": 6.0,
      "grad_norm": 1.6447800397872925,
      "learning_rate": 2.0658795558326745e-06,
      "logits/chosen": -0.36904090642929077,
      "logits/rejected": -0.43377822637557983,
      "logps/chosen": -0.05662701651453972,
      "logps/rejected": -0.6641786098480225,
      "loss": 0.0778,
      "odds_ratio_loss": 0.7118644714355469,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.005662702023983002,
      "rewards/margins": 0.060755155980587006,
      "rewards/rejected": -0.06641785800457001,
      "sft_loss": 0.006575153209269047,
      "step": 270
    },
    {
      "epoch": 6.222222222222222,
      "grad_norm": 2.223752021789551,
      "learning_rate": 1.876397139855047e-06,
      "logits/chosen": -0.3556548058986664,
      "logits/rejected": -0.4135330319404602,
      "logps/chosen": -0.0746704488992691,
      "logps/rejected": -0.6784394979476929,
      "loss": 0.1018,
      "odds_ratio_loss": 0.9240021705627441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007467044983059168,
      "rewards/margins": 0.06037690117955208,
      "rewards/rejected": -0.0678439512848854,
      "sft_loss": 0.009404648095369339,
      "step": 280
    },
    {
      "epoch": 6.444444444444445,
      "grad_norm": 1.75334894657135,
      "learning_rate": 1.6906651448541977e-06,
      "logits/chosen": -0.3690106272697449,
      "logits/rejected": -0.4269256591796875,
      "logps/chosen": -0.05620593950152397,
      "logps/rejected": -0.6589001417160034,
      "loss": 0.0805,
      "odds_ratio_loss": 0.7411076426506042,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.005620594136416912,
      "rewards/margins": 0.060269422829151154,
      "rewards/rejected": -0.06589001417160034,
      "sft_loss": 0.006437716074287891,
      "step": 290
    },
    {
      "epoch": 6.666666666666667,
      "grad_norm": 4.5506911277771,
      "learning_rate": 1.509800584902108e-06,
      "logits/chosen": -0.3519323468208313,
      "logits/rejected": -0.41471752524375916,
      "logps/chosen": -0.10020919144153595,
      "logps/rejected": -0.6135331392288208,
      "loss": 0.1385,
      "odds_ratio_loss": 1.263484001159668,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.010020919144153595,
      "rewards/margins": 0.051332395523786545,
      "rewards/rejected": -0.06135330721735954,
      "sft_loss": 0.012116280384361744,
      "step": 300
    },
    {
      "epoch": 6.666666666666667,
      "eval_logits/chosen": -0.32135990262031555,
      "eval_logits/rejected": -0.3725373148918152,
      "eval_logps/chosen": -0.12169794738292694,
      "eval_logps/rejected": -0.6881048083305359,
      "eval_loss": 0.15343782305717468,
      "eval_odds_ratio_loss": 1.3728852272033691,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": -0.012169795110821724,
      "eval_rewards/margins": 0.05664069205522537,
      "eval_rewards/rejected": -0.06881048530340195,
      "eval_runtime": 3.3342,
      "eval_samples_per_second": 23.994,
      "eval_sft_loss": 0.016149301081895828,
      "eval_steps_per_second": 11.997,
      "step": 300
    },
    {
      "epoch": 6.888888888888889,
      "grad_norm": 1.4514155387878418,
      "learning_rate": 1.3348912007436538e-06,
      "logits/chosen": -0.33361396193504333,
      "logits/rejected": -0.3964681923389435,
      "logps/chosen": -0.05151065066456795,
      "logps/rejected": -0.6935869455337524,
      "loss": 0.075,
      "odds_ratio_loss": 0.6938644051551819,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.005151065066456795,
      "rewards/margins": 0.06420762091875076,
      "rewards/rejected": -0.069358691573143,
      "sft_loss": 0.005607123486697674,
      "step": 310
    },
    {
      "epoch": 7.111111111111111,
      "grad_norm": 1.1120833158493042,
      "learning_rate": 1.1669889179957725e-06,
      "logits/chosen": -0.34897491335868835,
      "logits/rejected": -0.4280971884727478,
      "logps/chosen": -0.06502507627010345,
      "logps/rejected": -0.724215030670166,
      "loss": 0.0902,
      "odds_ratio_loss": 0.8162932395935059,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.006502507720142603,
      "rewards/margins": 0.06591899693012238,
      "rewards/rejected": -0.07242151349782944,
      "sft_loss": 0.008545474149286747,
      "step": 320
    },
    {
      "epoch": 7.333333333333333,
      "grad_norm": 0.9275766611099243,
      "learning_rate": 1.0071035207430352e-06,
      "logits/chosen": -0.34552061557769775,
      "logits/rejected": -0.4186931550502777,
      "logps/chosen": -0.057514738291502,
      "logps/rejected": -0.7304258942604065,
      "loss": 0.0812,
      "odds_ratio_loss": 0.7468503713607788,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.005751474294811487,
      "rewards/margins": 0.06729111820459366,
      "rewards/rejected": -0.07304258644580841,
      "sft_loss": 0.006484520621597767,
      "step": 330
    },
    {
      "epoch": 7.555555555555555,
      "grad_norm": 1.7526174783706665,
      "learning_rate": 8.561965785773413e-07,
      "logits/chosen": -0.35446950793266296,
      "logits/rejected": -0.41398343443870544,
      "logps/chosen": -0.04472974315285683,
      "logps/rejected": -0.6881515979766846,
      "loss": 0.0655,
      "odds_ratio_loss": 0.5992408394813538,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.004472973756492138,
      "rewards/margins": 0.0643421858549118,
      "rewards/rejected": -0.06881515681743622,
      "sft_loss": 0.0055578285828232765,
      "step": 340
    },
    {
      "epoch": 7.777777777777778,
      "grad_norm": 1.4603219032287598,
      "learning_rate": 7.151756636052529e-07,
      "logits/chosen": -0.35244011878967285,
      "logits/rejected": -0.4057006239891052,
      "logps/chosen": -0.06629346311092377,
      "logps/rejected": -0.6709402203559875,
      "loss": 0.0918,
      "odds_ratio_loss": 0.841974139213562,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.0066293468698859215,
      "rewards/margins": 0.060464680194854736,
      "rewards/rejected": -0.06709402799606323,
      "sft_loss": 0.0076185790821909904,
      "step": 350
    },
    {
      "epoch": 7.777777777777778,
      "eval_logits/chosen": -0.31914329528808594,
      "eval_logits/rejected": -0.3697816729545593,
      "eval_logps/chosen": -0.12174425274133682,
      "eval_logps/rejected": -0.6888757944107056,
      "eval_loss": 0.1536635458469391,
      "eval_odds_ratio_loss": 1.3741592168807983,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.012174425646662712,
      "eval_rewards/margins": 0.05671315640211105,
      "eval_rewards/rejected": -0.06888757646083832,
      "eval_runtime": 3.3381,
      "eval_samples_per_second": 23.966,
      "eval_sft_loss": 0.016247618943452835,
      "eval_steps_per_second": 11.983,
      "step": 350
    },
    {
      "epoch": 8.0,
      "grad_norm": 0.9194740056991577,
      "learning_rate": 5.848888922025553e-07,
      "logits/chosen": -0.3591064214706421,
      "logits/rejected": -0.4297390878200531,
      "logps/chosen": -0.08391134440898895,
      "logps/rejected": -0.7468418478965759,
      "loss": 0.118,
      "odds_ratio_loss": 1.0900821685791016,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.00839113537222147,
      "rewards/margins": 0.06629304587841034,
      "rewards/rejected": -0.07468418776988983,
      "sft_loss": 0.008979488164186478,
      "step": 360
    },
    {
      "epoch": 8.222222222222221,
      "grad_norm": 1.1211438179016113,
      "learning_rate": 4.661198243425813e-07,
      "logits/chosen": -0.3469446301460266,
      "logits/rejected": -0.416546493768692,
      "logps/chosen": -0.06295167654752731,
      "logps/rejected": -0.6733905076980591,
      "loss": 0.0877,
      "odds_ratio_loss": 0.7965422868728638,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.006295167841017246,
      "rewards/margins": 0.0610438771545887,
      "rewards/rejected": -0.06733904778957367,
      "sft_loss": 0.008044925518333912,
      "step": 370
    },
    {
      "epoch": 8.444444444444445,
      "grad_norm": 2.9023804664611816,
      "learning_rate": 3.595827511743341e-07,
      "logits/chosen": -0.36501026153564453,
      "logits/rejected": -0.4163384437561035,
      "logps/chosen": -0.0608031265437603,
      "logps/rejected": -0.7410011291503906,
      "loss": 0.0857,
      "odds_ratio_loss": 0.7958894371986389,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.006080311723053455,
      "rewards/margins": 0.068019799888134,
      "rewards/rejected": -0.07410011440515518,
      "sft_loss": 0.006128143519163132,
      "step": 380
    },
    {
      "epoch": 8.666666666666666,
      "grad_norm": 1.2382421493530273,
      "learning_rate": 2.6591839919146963e-07,
      "logits/chosen": -0.3580993711948395,
      "logits/rejected": -0.4169589579105377,
      "logps/chosen": -0.07410109043121338,
      "logps/rejected": -0.7789124250411987,
      "loss": 0.1054,
      "odds_ratio_loss": 0.9693442583084106,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0074101099744439125,
      "rewards/margins": 0.07048113644123077,
      "rewards/rejected": -0.07789124548435211,
      "sft_loss": 0.008436056785285473,
      "step": 390
    },
    {
      "epoch": 8.88888888888889,
      "grad_norm": 1.7495219707489014,
      "learning_rate": 1.8569007682777417e-07,
      "logits/chosen": -0.37370580434799194,
      "logits/rejected": -0.43429917097091675,
      "logps/chosen": -0.05244039371609688,
      "logps/rejected": -0.7023984789848328,
      "loss": 0.0752,
      "odds_ratio_loss": 0.6945605874061584,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.005244039464741945,
      "rewards/margins": 0.06499581038951874,
      "rewards/rejected": -0.0702398419380188,
      "sft_loss": 0.005755658261477947,
      "step": 400
    },
    {
      "epoch": 8.88888888888889,
      "eval_logits/chosen": -0.3194546103477478,
      "eval_logits/rejected": -0.3705773651599884,
      "eval_logps/chosen": -0.1213468462228775,
      "eval_logps/rejected": -0.689638614654541,
      "eval_loss": 0.15339744091033936,
      "eval_odds_ratio_loss": 1.3723235130310059,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": -0.01213468424975872,
      "eval_rewards/margins": 0.056829191744327545,
      "eval_rewards/rejected": -0.06896387040615082,
      "eval_runtime": 3.3351,
      "eval_samples_per_second": 23.987,
      "eval_sft_loss": 0.016165101900696754,
      "eval_steps_per_second": 11.994,
      "step": 400
    },
    {
      "epoch": 9.11111111111111,
      "grad_norm": 2.9914894104003906,
      "learning_rate": 1.1938028665396172e-07,
      "logits/chosen": -0.3595274090766907,
      "logits/rejected": -0.41625022888183594,
      "logps/chosen": -0.07939668744802475,
      "logps/rejected": -0.6182006597518921,
      "loss": 0.1064,
      "odds_ratio_loss": 0.9655207395553589,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.007939668372273445,
      "rewards/margins": 0.05388040468096733,
      "rewards/rejected": -0.06182006746530533,
      "sft_loss": 0.009880236349999905,
      "step": 410
    },
    {
      "epoch": 9.333333333333334,
      "grad_norm": 1.7583844661712646,
      "learning_rate": 6.738782355044048e-08,
      "logits/chosen": -0.35592731833457947,
      "logits/rejected": -0.41578736901283264,
      "logps/chosen": -0.06436184048652649,
      "logps/rejected": -0.6822584867477417,
      "loss": 0.0919,
      "odds_ratio_loss": 0.8448025584220886,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.006436183117330074,
      "rewards/margins": 0.06178967282176018,
      "rewards/rejected": -0.06822585314512253,
      "sft_loss": 0.007466461509466171,
      "step": 420
    },
    {
      "epoch": 9.555555555555555,
      "grad_norm": 0.7222520112991333,
      "learning_rate": 3.0025376307977474e-08,
      "logits/chosen": -0.3527710437774658,
      "logits/rejected": -0.41858869791030884,
      "logps/chosen": -0.042861782014369965,
      "logps/rejected": -0.7103925943374634,
      "loss": 0.0601,
      "odds_ratio_loss": 0.5496730804443359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.004286178387701511,
      "rewards/margins": 0.0667530745267868,
      "rewards/rejected": -0.07103925198316574,
      "sft_loss": 0.005130757577717304,
      "step": 430
    },
    {
      "epoch": 9.777777777777779,
      "grad_norm": 1.2489662170410156,
      "learning_rate": 7.517647080519941e-09,
      "logits/chosen": -0.34115296602249146,
      "logits/rejected": -0.39803385734558105,
      "logps/chosen": -0.05217872932553291,
      "logps/rejected": -0.8210271000862122,
      "loss": 0.0779,
      "odds_ratio_loss": 0.7251914739608765,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.0052178725600242615,
      "rewards/margins": 0.07688483595848083,
      "rewards/rejected": -0.0821027159690857,
      "sft_loss": 0.0053517939522862434,
      "step": 440
    },
    {
      "epoch": 10.0,
      "grad_norm": 1.9128329753875732,
      "learning_rate": 0.0,
      "logits/chosen": -0.3783785402774811,
      "logits/rejected": -0.4389713406562805,
      "logps/chosen": -0.07910243421792984,
      "logps/rejected": -0.6643815040588379,
      "loss": 0.1052,
      "odds_ratio_loss": 0.9564045667648315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007910243235528469,
      "rewards/margins": 0.05852789804339409,
      "rewards/rejected": -0.06643815338611603,
      "sft_loss": 0.009510872885584831,
      "step": 450
    },
    {
      "epoch": 10.0,
      "eval_logits/chosen": -0.3184443712234497,
      "eval_logits/rejected": -0.36903008818626404,
      "eval_logps/chosen": -0.1208958849310875,
      "eval_logps/rejected": -0.6922038793563843,
      "eval_loss": 0.1529119908809662,
      "eval_odds_ratio_loss": 1.3675727844238281,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.012089588679373264,
      "eval_rewards/margins": 0.05713079497218132,
      "eval_rewards/rejected": -0.06922038644552231,
      "eval_runtime": 3.3378,
      "eval_samples_per_second": 23.968,
      "eval_sft_loss": 0.016154715791344643,
      "eval_steps_per_second": 11.984,
      "step": 450
    },
    {
      "epoch": 10.0,
      "step": 450,
      "total_flos": 8.074648418018918e+16,
      "train_loss": 0.3428536836306254,
      "train_runtime": 994.0751,
      "train_samples_per_second": 7.243,
      "train_steps_per_second": 0.453
    }
  ],
  "logging_steps": 10,
  "max_steps": 450,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 8.074648418018918e+16,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}