{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 118.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": 88.18099975585938,
      "logits/rejected": 88.25153350830078,
      "logps/chosen": -29.073104858398438,
      "logps/rejected": -26.25731658935547,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 97.0,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": 81.08590698242188,
      "logits/rejected": 80.79015350341797,
      "logps/chosen": -34.172550201416016,
      "logps/rejected": -32.99056625366211,
      "loss": 0.9305,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.062255993485450745,
      "rewards/margins": 0.09120028465986252,
      "rewards/rejected": -0.028944293037056923,
      "step": 10
    },
    {
      "epoch": 0.05,
      "grad_norm": 83.5,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": 80.69185638427734,
      "logits/rejected": 80.58460235595703,
      "logps/chosen": -33.645851135253906,
      "logps/rejected": -30.73211097717285,
      "loss": 1.0725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.039649851620197296,
      "rewards/margins": -0.0011898368829861283,
      "rewards/rejected": 0.04083969444036484,
      "step": 20
    },
    {
      "epoch": 0.08,
      "grad_norm": 105.5,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": 82.52629089355469,
      "logits/rejected": 82.55845642089844,
      "logps/chosen": -33.839778900146484,
      "logps/rejected": -31.187374114990234,
      "loss": 1.1222,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.15573647618293762,
      "rewards/margins": -0.05707378312945366,
      "rewards/rejected": 0.21281024813652039,
      "step": 30
    },
    {
      "epoch": 0.1,
      "grad_norm": 83.5,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": 81.00019073486328,
      "logits/rejected": 80.99311828613281,
      "logps/chosen": -32.78485870361328,
      "logps/rejected": -33.173404693603516,
      "loss": 1.023,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.35028138756752014,
      "rewards/margins": 0.17467467486858368,
      "rewards/rejected": 0.17560675740242004,
      "step": 40
    },
    {
      "epoch": 0.13,
      "grad_norm": 72.5,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": 78.55744934082031,
      "logits/rejected": 78.57440948486328,
      "logps/chosen": -30.58083724975586,
      "logps/rejected": -30.754711151123047,
      "loss": 1.0219,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.5619677901268005,
      "rewards/margins": 0.2752513289451599,
      "rewards/rejected": 0.2867165207862854,
      "step": 50
    },
    {
      "epoch": 0.16,
      "grad_norm": 94.5,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": 83.01163482666016,
      "logits/rejected": 83.06632995605469,
      "logps/chosen": -30.960674285888672,
      "logps/rejected": -29.359949111938477,
      "loss": 1.1647,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.19303548336029053,
      "rewards/margins": -0.014480452053248882,
      "rewards/rejected": 0.20751595497131348,
      "step": 60
    },
    {
      "epoch": 0.18,
      "grad_norm": 116.5,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": 83.62989044189453,
      "logits/rejected": 83.66007232666016,
      "logps/chosen": -30.669225692749023,
      "logps/rejected": -33.12618637084961,
      "loss": 1.1523,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.03618011251091957,
      "rewards/margins": 0.04344985634088516,
      "rewards/rejected": -0.07962997257709503,
      "step": 70
    },
    {
      "epoch": 0.21,
      "grad_norm": 85.0,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": 81.133056640625,
      "logits/rejected": 81.11943054199219,
      "logps/chosen": -31.318958282470703,
      "logps/rejected": -30.970218658447266,
      "loss": 0.845,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.167384073138237,
      "rewards/margins": 0.3586970567703247,
      "rewards/rejected": -0.1913129985332489,
      "step": 80
    },
    {
      "epoch": 0.23,
      "grad_norm": 109.0,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": 77.96097564697266,
      "logits/rejected": 77.93636322021484,
      "logps/chosen": -32.162574768066406,
      "logps/rejected": -31.054067611694336,
      "loss": 0.7906,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.4280422329902649,
      "rewards/margins": 0.5203782916069031,
      "rewards/rejected": -0.09233605861663818,
      "step": 90
    },
    {
      "epoch": 0.26,
      "grad_norm": 68.5,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": 83.30989837646484,
      "logits/rejected": 83.33601379394531,
      "logps/chosen": -33.85996627807617,
      "logps/rejected": -31.64011573791504,
      "loss": 0.9852,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.4005257189273834,
      "rewards/margins": 0.30113959312438965,
      "rewards/rejected": 0.09938610345125198,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": 98.69417572021484,
      "eval_logits/rejected": 98.68303680419922,
      "eval_logps/chosen": -32.46882629394531,
      "eval_logps/rejected": -36.016597747802734,
      "eval_loss": 1.1082578897476196,
      "eval_rewards/accuracies": 0.4875415563583374,
      "eval_rewards/chosen": -0.02308560535311699,
      "eval_rewards/margins": 0.021978026255965233,
      "eval_rewards/rejected": -0.04506362974643707,
      "eval_runtime": 104.3924,
      "eval_samples_per_second": 3.286,
      "eval_steps_per_second": 0.412,
      "step": 100
    },
    {
      "epoch": 0.29,
      "grad_norm": 100.0,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": 83.43294525146484,
      "logits/rejected": 83.3353042602539,
      "logps/chosen": -32.524871826171875,
      "logps/rejected": -32.74653625488281,
      "loss": 0.8192,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4481372833251953,
      "rewards/margins": 0.5297205448150635,
      "rewards/rejected": -0.08158326148986816,
      "step": 110
    },
    {
      "epoch": 0.31,
      "grad_norm": 86.5,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": 83.61913299560547,
      "logits/rejected": 83.73504638671875,
      "logps/chosen": -28.402780532836914,
      "logps/rejected": -35.50286865234375,
      "loss": 0.7691,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.4741190969944,
      "rewards/margins": 0.5134681463241577,
      "rewards/rejected": -0.039348993450403214,
      "step": 120
    },
    {
      "epoch": 0.34,
      "grad_norm": 72.0,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": 80.9137954711914,
      "logits/rejected": 80.93389892578125,
      "logps/chosen": -30.521535873413086,
      "logps/rejected": -32.07093048095703,
      "loss": 0.815,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.34723615646362305,
      "rewards/margins": 0.47260579466819763,
      "rewards/rejected": -0.12536963820457458,
      "step": 130
    },
    {
      "epoch": 0.36,
      "grad_norm": 86.0,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": 82.1353530883789,
      "logits/rejected": 82.16053771972656,
      "logps/chosen": -27.094701766967773,
      "logps/rejected": -32.902889251708984,
      "loss": 0.7235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.3173602223396301,
      "rewards/margins": 0.7489217519760132,
      "rewards/rejected": -0.4315616488456726,
      "step": 140
    },
    {
      "epoch": 0.39,
      "grad_norm": 59.75,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": 80.8051986694336,
      "logits/rejected": 80.77262115478516,
      "logps/chosen": -28.932445526123047,
      "logps/rejected": -33.024757385253906,
      "loss": 0.633,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.4109979569911957,
      "rewards/margins": 0.8234249949455261,
      "rewards/rejected": -0.4124270975589752,
      "step": 150
    },
    {
      "epoch": 0.42,
      "grad_norm": 82.5,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": 82.64894104003906,
      "logits/rejected": 82.64643859863281,
      "logps/chosen": -33.55781936645508,
      "logps/rejected": -30.31070899963379,
      "loss": 0.7859,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.5210397839546204,
      "rewards/margins": 0.8433935046195984,
      "rewards/rejected": -0.3223537802696228,
      "step": 160
    },
    {
      "epoch": 0.44,
      "grad_norm": 81.5,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": 83.43087005615234,
      "logits/rejected": 83.37384033203125,
      "logps/chosen": -30.71698570251465,
      "logps/rejected": -32.415626525878906,
      "loss": 0.6671,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.4880717396736145,
      "rewards/margins": 0.8440794944763184,
      "rewards/rejected": -0.35600775480270386,
      "step": 170
    },
    {
      "epoch": 0.47,
      "grad_norm": 64.0,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": 81.02064514160156,
      "logits/rejected": 81.00648498535156,
      "logps/chosen": -30.46847152709961,
      "logps/rejected": -31.655284881591797,
      "loss": 0.6529,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.6558108925819397,
      "rewards/margins": 1.104081153869629,
      "rewards/rejected": -0.4482702314853668,
      "step": 180
    },
    {
      "epoch": 0.49,
      "grad_norm": 27.25,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": 82.70450592041016,
      "logits/rejected": 82.70726776123047,
      "logps/chosen": -30.44081687927246,
      "logps/rejected": -30.764562606811523,
      "loss": 0.8467,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2930375039577484,
      "rewards/margins": 0.5599225759506226,
      "rewards/rejected": -0.2668851315975189,
      "step": 190
    },
    {
      "epoch": 0.52,
      "grad_norm": 68.0,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": 78.14289093017578,
      "logits/rejected": 78.0774917602539,
      "logps/chosen": -34.14347839355469,
      "logps/rejected": -32.717750549316406,
      "loss": 0.902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5483377575874329,
      "rewards/margins": 0.7502498030662537,
      "rewards/rejected": -0.2019120752811432,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": 98.65617370605469,
      "eval_logits/rejected": 98.62713623046875,
      "eval_logps/chosen": -32.60585021972656,
      "eval_logps/rejected": -36.384361267089844,
      "eval_loss": 0.9846288561820984,
      "eval_rewards/accuracies": 0.554817259311676,
      "eval_rewards/chosen": -0.14640627801418304,
      "eval_rewards/margins": 0.22964100539684296,
      "eval_rewards/rejected": -0.3760472238063812,
      "eval_runtime": 103.9231,
      "eval_samples_per_second": 3.301,
      "eval_steps_per_second": 0.414,
      "step": 200
    },
    {
      "epoch": 0.55,
      "grad_norm": 107.0,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": 80.76277160644531,
      "logits/rejected": 80.67192840576172,
      "logps/chosen": -33.424560546875,
      "logps/rejected": -35.27748107910156,
      "loss": 0.7859,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4690447747707367,
      "rewards/margins": 0.7312911152839661,
      "rewards/rejected": -0.26224634051322937,
      "step": 210
    },
    {
      "epoch": 0.57,
      "grad_norm": 86.0,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": 82.9009017944336,
      "logits/rejected": 82.97982788085938,
      "logps/chosen": -31.129741668701172,
      "logps/rejected": -31.022235870361328,
      "loss": 0.566,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7372905611991882,
      "rewards/margins": 1.001245141029358,
      "rewards/rejected": -0.2639545798301697,
      "step": 220
    },
    {
      "epoch": 0.6,
      "grad_norm": 79.0,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": 80.10737609863281,
      "logits/rejected": 80.16769409179688,
      "logps/chosen": -32.2999153137207,
      "logps/rejected": -34.19389343261719,
      "loss": 0.904,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.45524734258651733,
      "rewards/margins": 0.6188174486160278,
      "rewards/rejected": -0.16357013583183289,
      "step": 230
    },
    {
      "epoch": 0.62,
      "grad_norm": 101.0,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": 82.58370208740234,
      "logits/rejected": 82.85044860839844,
      "logps/chosen": -30.799701690673828,
      "logps/rejected": -31.604589462280273,
      "loss": 0.7596,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.7995287179946899,
      "rewards/margins": 0.8648217916488647,
      "rewards/rejected": -0.06529306620359421,
      "step": 240
    },
    {
      "epoch": 0.65,
      "grad_norm": 89.5,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": 81.37136840820312,
      "logits/rejected": 81.4355697631836,
      "logps/chosen": -26.921367645263672,
      "logps/rejected": -30.0640811920166,
      "loss": 0.8158,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.6070747971534729,
      "rewards/margins": 0.754082202911377,
      "rewards/rejected": -0.14700737595558167,
      "step": 250
    },
    {
      "epoch": 0.68,
      "grad_norm": 69.0,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": 78.6139144897461,
      "logits/rejected": 78.74816131591797,
      "logps/chosen": -30.19488525390625,
      "logps/rejected": -36.41002655029297,
      "loss": 0.4662,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.2295000553131104,
      "rewards/margins": 1.5070557594299316,
      "rewards/rejected": -0.2775557041168213,
      "step": 260
    },
    {
      "epoch": 0.7,
      "grad_norm": 36.0,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": 78.08236694335938,
      "logits/rejected": 78.10992431640625,
      "logps/chosen": -30.87982749938965,
      "logps/rejected": -31.692874908447266,
      "loss": 0.6864,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.8091908693313599,
      "rewards/margins": 0.977896511554718,
      "rewards/rejected": -0.16870568692684174,
      "step": 270
    },
    {
      "epoch": 0.73,
      "grad_norm": 113.0,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": 80.7326431274414,
      "logits/rejected": 80.51844024658203,
      "logps/chosen": -31.088062286376953,
      "logps/rejected": -29.66598892211914,
      "loss": 0.9291,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.6496511697769165,
      "rewards/margins": 0.6487425565719604,
      "rewards/rejected": 0.0009086370700970292,
      "step": 280
    },
    {
      "epoch": 0.75,
      "grad_norm": 72.0,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": 80.9128189086914,
      "logits/rejected": 80.82886505126953,
      "logps/chosen": -33.01869201660156,
      "logps/rejected": -32.421485900878906,
      "loss": 0.5234,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.977002739906311,
      "rewards/margins": 1.3411604166030884,
      "rewards/rejected": -0.3641577661037445,
      "step": 290
    },
    {
      "epoch": 0.78,
      "grad_norm": 70.5,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": 76.50440979003906,
      "logits/rejected": 76.59843444824219,
      "logps/chosen": -32.13983917236328,
      "logps/rejected": -29.128122329711914,
      "loss": 0.671,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.139281988143921,
      "rewards/margins": 1.1627700328826904,
      "rewards/rejected": -0.023488014936447144,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": 98.7301254272461,
      "eval_logits/rejected": 98.71080017089844,
      "eval_logps/chosen": -32.50548553466797,
      "eval_logps/rejected": -36.0523567199707,
      "eval_loss": 1.1081271171569824,
      "eval_rewards/accuracies": 0.47757473587989807,
      "eval_rewards/chosen": -0.056078068912029266,
      "eval_rewards/margins": 0.021167948842048645,
      "eval_rewards/rejected": -0.07724600285291672,
      "eval_runtime": 103.8638,
      "eval_samples_per_second": 3.302,
      "eval_steps_per_second": 0.414,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 66.0,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": 83.55355072021484,
      "logits/rejected": 83.5801773071289,
      "logps/chosen": -30.023509979248047,
      "logps/rejected": -32.43012237548828,
      "loss": 0.6531,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.7560194730758667,
      "rewards/margins": 0.9633440971374512,
      "rewards/rejected": -0.20732466876506805,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 56.25,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": 81.07228088378906,
      "logits/rejected": 81.07295989990234,
      "logps/chosen": -30.569133758544922,
      "logps/rejected": -29.126922607421875,
      "loss": 0.5552,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.926487147808075,
      "rewards/margins": 1.1131722927093506,
      "rewards/rejected": -0.1866852045059204,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 65.5,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": 78.3108139038086,
      "logits/rejected": 78.35076904296875,
      "logps/chosen": -29.139026641845703,
      "logps/rejected": -32.85981750488281,
      "loss": 0.5897,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 1.0922209024429321,
      "rewards/margins": 1.294679045677185,
      "rewards/rejected": -0.20245835185050964,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 98.5,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": 82.51289367675781,
      "logits/rejected": 82.52921295166016,
      "logps/chosen": -32.200294494628906,
      "logps/rejected": -33.6099853515625,
      "loss": 0.7511,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.910667896270752,
      "rewards/margins": 1.127828598022461,
      "rewards/rejected": -0.21716061234474182,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 47.0,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": 81.58796691894531,
      "logits/rejected": 81.5913314819336,
      "logps/chosen": -32.477813720703125,
      "logps/rejected": -33.15494155883789,
      "loss": 0.6388,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 1.049329400062561,
      "rewards/margins": 1.106154203414917,
      "rewards/rejected": -0.056824732571840286,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 59.0,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": 83.0643539428711,
      "logits/rejected": 83.09364318847656,
      "logps/chosen": -28.514789581298828,
      "logps/rejected": -31.751323699951172,
      "loss": 0.5731,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.9336811304092407,
      "rewards/margins": 1.0250084400177002,
      "rewards/rejected": -0.09132737666368484,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 121.5,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": 82.48683166503906,
      "logits/rejected": 82.50460815429688,
      "logps/chosen": -31.874202728271484,
      "logps/rejected": -35.20905303955078,
      "loss": 0.7031,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.9017645120620728,
      "rewards/margins": 0.9914749264717102,
      "rewards/rejected": -0.08971036225557327,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 94.5,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": 76.47808074951172,
      "logits/rejected": 76.34886932373047,
      "logps/chosen": -29.865795135498047,
      "logps/rejected": -28.233165740966797,
      "loss": 0.7695,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.6392983794212341,
      "rewards/margins": 0.7046308517456055,
      "rewards/rejected": -0.06533239781856537,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.7878170496457583,
      "train_runtime": 2553.4781,
      "train_samples_per_second": 1.206,
      "train_steps_per_second": 0.151
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}