{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.5102,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.866772174835205,
      "logits/rejected": -1.8710780143737793,
      "logps/chosen": -36.99943161010742,
      "logps/rejected": -33.657470703125,
      "loss": 0.4858,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.005016062408685684,
      "rewards/margins": 0.02419126406311989,
      "rewards/rejected": -0.019175197929143906,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9975645542144775,
      "logits/rejected": -2.0002217292785645,
      "logps/chosen": -29.651592254638672,
      "logps/rejected": -29.066898345947266,
      "loss": 0.5542,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006579822860658169,
      "rewards/margins": -0.004162783268839121,
      "rewards/rejected": -0.0024170405231416225,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9212992191314697,
      "logits/rejected": -1.918600082397461,
      "logps/chosen": -31.42719078063965,
      "logps/rejected": -33.23390579223633,
      "loss": 0.5492,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.00776728754863143,
      "rewards/margins": 0.0030429032631218433,
      "rewards/rejected": -0.010810190811753273,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0178468227386475,
      "logits/rejected": -2.009111166000366,
      "logps/chosen": -32.579471588134766,
      "logps/rejected": -32.50147247314453,
      "loss": 0.5878,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.001865685684606433,
      "rewards/margins": -0.009705344215035439,
      "rewards/rejected": 0.007839656434953213,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8617979288101196,
      "logits/rejected": -1.8510282039642334,
      "logps/chosen": -33.562862396240234,
      "logps/rejected": -35.4472770690918,
      "loss": 0.5775,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0037666684947907925,
      "rewards/margins": -0.007314275950193405,
      "rewards/rejected": 0.0035476074554026127,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9399009943008423,
      "logits/rejected": -1.9418485164642334,
      "logps/chosen": -32.5726203918457,
      "logps/rejected": -33.18496322631836,
      "loss": 0.5967,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.019294817000627518,
      "rewards/margins": 0.028803948312997818,
      "rewards/rejected": -0.009509134106338024,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.0729470252990723,
      "logits/rejected": -2.077918529510498,
      "logps/chosen": -33.93268585205078,
      "logps/rejected": -36.637229919433594,
      "loss": 0.5297,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.039063483476638794,
      "rewards/margins": 0.08691467344760895,
      "rewards/rejected": -0.04785118252038956,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9366486072540283,
      "logits/rejected": -1.939780592918396,
      "logps/chosen": -34.30228805541992,
      "logps/rejected": -34.652069091796875,
      "loss": 0.4209,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.08786438405513763,
      "rewards/margins": 0.12962770462036133,
      "rewards/rejected": -0.041763320565223694,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.946979284286499,
      "logits/rejected": -1.951521635055542,
      "logps/chosen": -32.4210205078125,
      "logps/rejected": -32.337799072265625,
      "loss": 0.6191,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.03248247504234314,
      "rewards/margins": 0.017702888697385788,
      "rewards/rejected": 0.014779585413634777,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.045017719268799,
      "logits/rejected": -2.043015241622925,
      "logps/chosen": -32.16361618041992,
      "logps/rejected": -31.273128509521484,
      "loss": 0.4711,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.05405203625559807,
      "rewards/margins": 0.05889582633972168,
      "rewards/rejected": -0.004843792412430048,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.240097761154175,
      "eval_logits/rejected": -2.2352383136749268,
      "eval_logps/chosen": -34.01129150390625,
      "eval_logps/rejected": -37.49793243408203,
      "eval_loss": 0.5754798054695129,
      "eval_rewards/accuracies": 0.5195183157920837,
      "eval_rewards/chosen": 0.016279777511954308,
      "eval_rewards/margins": 0.0031977586913853884,
      "eval_rewards/rejected": 0.013082021847367287,
      "eval_runtime": 145.9275,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -2.0017762184143066,
      "logits/rejected": -1.9993689060211182,
      "logps/chosen": -33.13564682006836,
      "logps/rejected": -33.997642517089844,
      "loss": 0.6759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07571890205144882,
      "rewards/margins": 0.05350743606686592,
      "rewards/rejected": 0.022211460396647453,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.013158082962036,
      "logits/rejected": -2.004814624786377,
      "logps/chosen": -32.32666015625,
      "logps/rejected": -32.12488555908203,
      "loss": 0.5572,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.08277235180139542,
      "rewards/margins": 0.05714136362075806,
      "rewards/rejected": 0.02563098631799221,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.039836883544922,
      "logits/rejected": -2.031825542449951,
      "logps/chosen": -30.349987030029297,
      "logps/rejected": -32.029136657714844,
      "loss": 0.5109,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0880090743303299,
      "rewards/margins": 0.07130132615566254,
      "rewards/rejected": 0.016707751899957657,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9694139957427979,
      "logits/rejected": -1.9796711206436157,
      "logps/chosen": -31.24441909790039,
      "logps/rejected": -32.56440353393555,
      "loss": 0.5037,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.12266886234283447,
      "rewards/margins": 0.14333835244178772,
      "rewards/rejected": -0.02066950313746929,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8828538656234741,
      "logits/rejected": -1.8840004205703735,
      "logps/chosen": -34.006561279296875,
      "logps/rejected": -34.81753921508789,
      "loss": 0.4564,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13028928637504578,
      "rewards/margins": 0.17248663306236267,
      "rewards/rejected": -0.04219735041260719,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9349613189697266,
      "logits/rejected": -1.9315464496612549,
      "logps/chosen": -36.059181213378906,
      "logps/rejected": -32.72267150878906,
      "loss": 0.487,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.07874591648578644,
      "rewards/margins": 0.07869003713130951,
      "rewards/rejected": 5.587190389633179e-05,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.036032199859619,
      "logits/rejected": -2.028651714324951,
      "logps/chosen": -33.573509216308594,
      "logps/rejected": -31.36956214904785,
      "loss": 0.4465,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.15704451501369476,
      "rewards/margins": 0.16944527626037598,
      "rewards/rejected": -0.012400749139487743,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.042617082595825,
      "logits/rejected": -2.0478556156158447,
      "logps/chosen": -32.33074188232422,
      "logps/rejected": -32.472496032714844,
      "loss": 0.3727,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.14983226358890533,
      "rewards/margins": 0.14098060131072998,
      "rewards/rejected": 0.008851657621562481,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0445618629455566,
      "logits/rejected": -2.0417864322662354,
      "logps/chosen": -31.317855834960938,
      "logps/rejected": -31.348400115966797,
      "loss": 0.4609,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.11976919323205948,
      "rewards/margins": 0.14090058207511902,
      "rewards/rejected": -0.021131375804543495,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9139589071273804,
      "logits/rejected": -1.9186115264892578,
      "logps/chosen": -31.420101165771484,
      "logps/rejected": -32.79216766357422,
      "loss": 0.5061,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.13032536208629608,
      "rewards/margins": 0.13119173049926758,
      "rewards/rejected": -0.0008663811022415757,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2385354042053223,
      "eval_logits/rejected": -2.2336885929107666,
      "eval_logps/chosen": -34.05002975463867,
      "eval_logps/rejected": -37.54545974731445,
      "eval_loss": 0.5877167582511902,
      "eval_rewards/accuracies": 0.4991694688796997,
      "eval_rewards/chosen": -0.010832725092768669,
      "eval_rewards/margins": 0.009353035129606724,
      "eval_rewards/rejected": -0.020185761153697968,
      "eval_runtime": 145.6958,
      "eval_samples_per_second": 2.354,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.026780366897583,
      "logits/rejected": -2.0374534130096436,
      "logps/chosen": -31.786846160888672,
      "logps/rejected": -33.922447204589844,
      "loss": 0.4225,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.12911871075630188,
      "rewards/margins": 0.17094141244888306,
      "rewards/rejected": -0.041822709143161774,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9182548522949219,
      "logits/rejected": -1.933038353919983,
      "logps/chosen": -29.936147689819336,
      "logps/rejected": -31.559993743896484,
      "loss": 0.4429,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.12277615070343018,
      "rewards/margins": 0.12404797971248627,
      "rewards/rejected": -0.0012718416983261704,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9737575054168701,
      "logits/rejected": -1.9777438640594482,
      "logps/chosen": -33.20977783203125,
      "logps/rejected": -31.5734920501709,
      "loss": 0.3768,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.14538228511810303,
      "rewards/margins": 0.16231416165828705,
      "rewards/rejected": -0.016931891441345215,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9738047122955322,
      "logits/rejected": -1.9518985748291016,
      "logps/chosen": -33.94301223754883,
      "logps/rejected": -35.023075103759766,
      "loss": 0.4304,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.1475195735692978,
      "rewards/margins": 0.19811663031578064,
      "rewards/rejected": -0.05059707164764404,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0155911445617676,
      "logits/rejected": -2.0122878551483154,
      "logps/chosen": -32.76276397705078,
      "logps/rejected": -36.25225830078125,
      "loss": 0.4155,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.11361332982778549,
      "rewards/margins": 0.1327473670244217,
      "rewards/rejected": -0.019134048372507095,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8830368518829346,
      "logits/rejected": -1.880629539489746,
      "logps/chosen": -34.01710891723633,
      "logps/rejected": -35.53314971923828,
      "loss": 0.4219,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.11990444362163544,
      "rewards/margins": 0.1398761123418808,
      "rewards/rejected": -0.019971664994955063,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8685131072998047,
      "logits/rejected": -1.8660367727279663,
      "logps/chosen": -34.2363395690918,
      "logps/rejected": -31.768321990966797,
      "loss": 0.4918,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.11288031190633774,
      "rewards/margins": 0.11077453196048737,
      "rewards/rejected": 0.0021057710982859135,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9722087383270264,
      "logits/rejected": -1.961612343788147,
      "logps/chosen": -35.04926300048828,
      "logps/rejected": -31.826486587524414,
      "loss": 0.3417,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.21516695618629456,
      "rewards/margins": 0.20501188933849335,
      "rewards/rejected": 0.010155050083994865,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.068380832672119,
      "logits/rejected": -2.05340313911438,
      "logps/chosen": -30.748706817626953,
      "logps/rejected": -32.63469314575195,
      "loss": 0.4741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12279413640499115,
      "rewards/margins": 0.11079368740320206,
      "rewards/rejected": 0.012000440619885921,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9397351741790771,
      "logits/rejected": -1.9371974468231201,
      "logps/chosen": -32.603660583496094,
      "logps/rejected": -30.863941192626953,
      "loss": 0.3371,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.2264125794172287,
      "rewards/margins": 0.25940603017807007,
      "rewards/rejected": -0.03299345448613167,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2370686531066895,
      "eval_logits/rejected": -2.2322022914886475,
      "eval_logps/chosen": -34.03439712524414,
      "eval_logps/rejected": -37.53529357910156,
      "eval_loss": 0.5842657089233398,
      "eval_rewards/accuracies": 0.5278239250183105,
      "eval_rewards/chosen": 0.00010742468293756247,
      "eval_rewards/margins": 0.013178782537579536,
      "eval_rewards/rejected": -0.013071359135210514,
      "eval_runtime": 145.5131,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.296,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.9219080209732056,
      "logits/rejected": -1.9186557531356812,
      "logps/chosen": -31.35263442993164,
      "logps/rejected": -33.746925354003906,
      "loss": 0.4182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17550265789031982,
      "rewards/margins": 0.17884370684623718,
      "rewards/rejected": -0.0033410280011594296,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.9732515811920166,
      "logits/rejected": -1.9609721899032593,
      "logps/chosen": -34.435001373291016,
      "logps/rejected": -33.593482971191406,
      "loss": 0.3778,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.11274988949298859,
      "rewards/margins": 0.15099774301052094,
      "rewards/rejected": -0.03824785351753235,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -2.0085272789001465,
      "logits/rejected": -2.0070912837982178,
      "logps/chosen": -33.31962585449219,
      "logps/rejected": -32.51466751098633,
      "loss": 0.4061,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1163158044219017,
      "rewards/margins": 0.14087431132793427,
      "rewards/rejected": -0.024558518081903458,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.0951385498046875,
      "logits/rejected": -2.0793802738189697,
      "logps/chosen": -33.88261032104492,
      "logps/rejected": -33.10671615600586,
      "loss": 0.4042,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.20774619281291962,
      "rewards/margins": 0.19982238113880157,
      "rewards/rejected": 0.00792380329221487,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.9677631855010986,
      "logits/rejected": -1.9668546915054321,
      "logps/chosen": -32.957977294921875,
      "logps/rejected": -32.483890533447266,
      "loss": 0.4874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2268829345703125,
      "rewards/margins": 0.2400023490190506,
      "rewards/rejected": -0.013119394890964031,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9235498905181885,
      "logits/rejected": -1.933876395225525,
      "logps/chosen": -32.016578674316406,
      "logps/rejected": -35.32080841064453,
      "loss": 0.4297,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.13167943060398102,
      "rewards/margins": 0.15411263704299927,
      "rewards/rejected": -0.02243318222463131,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.062828540802002,
      "logits/rejected": -2.056312322616577,
      "logps/chosen": -33.43348693847656,
      "logps/rejected": -29.213970184326172,
      "loss": 0.3963,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.15293903648853302,
      "rewards/margins": 0.14422088861465454,
      "rewards/rejected": 0.008718125522136688,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9224169254302979,
      "logits/rejected": -1.9245796203613281,
      "logps/chosen": -33.9437255859375,
      "logps/rejected": -30.860538482666016,
      "loss": 0.4119,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.20672598481178284,
      "rewards/margins": 0.19826285541057587,
      "rewards/rejected": 0.008463130332529545,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.46899499707407766,
      "train_runtime": 3249.86,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}