Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4992
-- Rewards/chosen: 0.0239
-- Rewards/rejected: 0.0174
-- Rewards/accuracies: 0.4892
-- Rewards/margins: 0.0064
-- Logps/rejected: -35.7921
-- Logps/chosen: -32.2044
-- Logits/rejected: 98.2668
-- Logits/chosen: 98.2727
+- Loss: 0.4855
+- Rewards/chosen: 0.1659
+- Rewards/rejected: 0.0986
+- Rewards/accuracies: 0.5282
+- Rewards/margins: 0.0673
+- Logps/rejected: -35.8433
+- Logps/chosen: -32.2358
+- Logits/rejected: 98.4267
+- Logits/chosen: 98.4293
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4933        | 0.26  | 100  | 0.5023          | 0.0062         | 0.0168           | 0.4498             | -0.0106         | -35.7990       | -32.3816     | 98.6627         | 98.6788       |
-| 0.4602        | 0.52  | 200  | 0.4967          | 0.0067         | -0.0120          | 0.5511             | 0.0188          | -36.0870       | -32.3759     | 98.3410         | 98.3552       |
-| 0.4586        | 0.78  | 300  | 0.4994          | 0.0186         | 0.0126           | 0.5129             | 0.0060          | -35.8407       | -32.2572     | 98.2712         | 98.2791       |
+| 0.4705        | 0.26  | 100  | 0.4947          | 0.1342         | 0.1226           | 0.4896             | 0.0116          | -35.8133       | -32.2754     | 98.6750         | 98.6871       |
+| 0.3097        | 0.52  | 200  | 0.4910          | 0.1604         | 0.1008           | 0.5482             | 0.0596          | -35.8405       | -32.2427     | 98.4702         | 98.4771       |
+| 0.3066        | 0.78  | 300  | 0.4855          | 0.1659         | 0.0986           | 0.5282             | 0.0673          | -35.8433       | -32.2358     | 98.4267         | 98.4293       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5cc8952d83395c510ac63fdbd76fc409103384be7aa6c36e2adbad4aa9c0e783
+oid sha256:bf079650c439cdb1f37ec5806ba619b31a174f7b193b1d7b513c190631ec4ff1
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.27266693115234,
-    "eval_logits/rejected": 98.26683044433594,
-    "eval_logps/chosen": -32.204383850097656,
-    "eval_logps/rejected": -35.79214859008789,
-    "eval_loss": 0.49918678402900696,
-    "eval_rewards/accuracies": 0.489202618598938,
-    "eval_rewards/chosen": 0.023878853768110275,
-    "eval_rewards/margins": 0.00644117034971714,
-    "eval_rewards/rejected": 0.017437683418393135,
-    "eval_runtime": 103.7538,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.306,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.47652822160101554,
-    "train_runtime": 2558.0485,
+    "train_loss": 0.37664644687206716,
+    "train_runtime": 2560.444,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
-    "train_steps_per_second": 0.151
+    "train_samples_per_second": 1.203,
+    "train_steps_per_second": 0.15
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.47652822160101554,
-    "train_runtime": 2558.0485,
+    "train_loss": 0.37664644687206716,
+    "train_runtime": 2560.444,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
-    "train_steps_per_second": 0.151
+    "train_samples_per_second": 1.203,
+    "train_steps_per_second": 0.15
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 3.265625,
+      "grad_norm": 26.125,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 1.625,
+      "grad_norm": 14.1875,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.08853912353516,
-      "logits/rejected": 80.79112243652344,
-      "logps/chosen": -34.252471923828125,
-      "logps/rejected": -32.9941291809082,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": -0.0010747427586466074,
-      "rewards/margins": 0.0024977123830467463,
-      "rewards/rejected": -0.003572455607354641,
+      "logits/chosen": 81.0709228515625,
+      "logits/rejected": 80.77515411376953,
+      "logps/chosen": -34.26983642578125,
+      "logps/rejected": -32.992767333984375,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.022488469257950783,
+      "rewards/margins": 0.004999637603759766,
+      "rewards/rejected": -0.02748810313642025,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 2.546875,
+      "grad_norm": 19.5,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.66322326660156,
-      "logits/rejected": 80.55162048339844,
-      "logps/chosen": -33.64655303955078,
-      "logps/rejected": -30.839855194091797,
-      "loss": 0.4982,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.004335404373705387,
-      "rewards/margins": 0.010572222992777824,
-      "rewards/rejected": -0.006236819084733725,
+      "logits/chosen": 80.6608657836914,
+      "logits/rejected": 80.54856872558594,
+      "logps/chosen": -33.62510299682617,
+      "logps/rejected": -30.784603118896484,
+      "loss": 0.4924,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.05184358358383179,
+      "rewards/margins": 0.057535119354724884,
+      "rewards/rejected": -0.00569154042750597,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 2.140625,
+      "grad_norm": 22.125,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.4841079711914,
-      "logits/rejected": 82.51509094238281,
-      "logps/chosen": -33.830867767333984,
-      "logps/rejected": -31.266870498657227,
-      "loss": 0.4988,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.018195267766714096,
-      "rewards/margins": 0.002499166876077652,
-      "rewards/rejected": 0.015696100890636444,
+      "logits/chosen": 82.50384521484375,
+      "logits/rejected": 82.53746032714844,
+      "logps/chosen": -33.812801361083984,
+      "logps/rejected": -31.15009117126465,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.16001427173614502,
+      "rewards/margins": -0.058978963643312454,
+      "rewards/rejected": 0.21899321675300598,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 2.90625,
+      "grad_norm": 18.0,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.04556274414062,
-      "logits/rejected": 81.04271697998047,
-      "logps/chosen": -32.56133270263672,
-      "logps/rejected": -33.02995300292969,
-      "loss": 0.4932,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.06127279996871948,
-      "rewards/margins": 0.0274154394865036,
-      "rewards/rejected": 0.03385736048221588,
+      "logits/chosen": 81.07115173339844,
+      "logits/rejected": 81.06380462646484,
+      "logps/chosen": -32.71454620361328,
+      "logps/rejected": -33.17704772949219,
+      "loss": 0.4627,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.36761564016342163,
+      "rewards/margins": 0.21443231403827667,
+      "rewards/rejected": 0.15318328142166138,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 2.84375,
+      "grad_norm": 16.75,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.80296325683594,
-      "logits/rejected": 78.81227111816406,
-      "logps/chosen": -30.36850929260254,
-      "logps/rejected": -30.71665382385254,
-      "loss": 0.4889,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.08367374539375305,
-      "rewards/margins": 0.04801047220826149,
-      "rewards/rejected": 0.03566327691078186,
+      "logits/chosen": 78.77976989746094,
+      "logits/rejected": 78.78811645507812,
+      "logps/chosen": -30.60152816772461,
+      "logps/rejected": -30.760879516601562,
+      "loss": 0.4673,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.4829755425453186,
+      "rewards/margins": 0.2330504208803177,
+      "rewards/rejected": 0.24992509186267853,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 1.8984375,
+      "grad_norm": 15.125,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.3783187866211,
-      "logits/rejected": 83.4344482421875,
-      "logps/chosen": -30.705408096313477,
-      "logps/rejected": -29.251195907592773,
-      "loss": 0.4967,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0469752736389637,
-      "rewards/margins": 0.013043051585555077,
-      "rewards/rejected": 0.033932216465473175,
+      "logits/chosen": 83.38993072509766,
+      "logits/rejected": 83.4462890625,
+      "logps/chosen": -30.877517700195312,
+      "logps/rejected": -29.332889556884766,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.23811349272727966,
+      "rewards/margins": 0.03200835734605789,
+      "rewards/rejected": 0.20610514283180237,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 2.828125,
+      "grad_norm": 15.6875,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 83.97593688964844,
-      "logits/rejected": 84.0093765258789,
-      "logps/chosen": -30.22530746459961,
-      "logps/rejected": -32.46759033203125,
-      "loss": 0.5037,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 0.0403718426823616,
-      "rewards/margins": -0.016639459878206253,
-      "rewards/rejected": 0.05701129883527756,
+      "logits/chosen": 84.03268432617188,
+      "logits/rejected": 84.06501007080078,
+      "logps/chosen": -30.23248291015625,
+      "logps/rejected": -32.74273681640625,
+      "loss": 0.4844,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.31723442673683167,
+      "rewards/margins": 0.08125714957714081,
+      "rewards/rejected": 0.23597724735736847,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 2.78125,
+      "grad_norm": 17.625,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.67536163330078,
-      "logits/rejected": 81.65579986572266,
-      "logps/chosen": -31.098180770874023,
-      "logps/rejected": -30.70516014099121,
-      "loss": 0.4922,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.040676504373550415,
-      "rewards/margins": 0.035427749156951904,
-      "rewards/rejected": 0.005248754285275936,
+      "logits/chosen": 81.7315673828125,
+      "logits/rejected": 81.71873474121094,
+      "logps/chosen": -30.963665008544922,
+      "logps/rejected": -30.75851058959961,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.4330216348171234,
+      "rewards/margins": 0.4337124824523926,
+      "rewards/rejected": -0.0006908595678396523,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 3.375,
+      "grad_norm": 21.25,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.46135711669922,
-      "logits/rejected": 78.43350982666016,
-      "logps/chosen": -32.169097900390625,
-      "logps/rejected": -30.87672996520996,
-      "loss": 0.4912,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.04690824821591377,
-      "rewards/margins": 0.03943391516804695,
-      "rewards/rejected": 0.007474330719560385,
+      "logits/chosen": 78.5487060546875,
+      "logits/rejected": 78.52108001708984,
+      "logps/chosen": -32.06660461425781,
+      "logps/rejected": -30.661907196044922,
+      "loss": 0.4704,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.4572601318359375,
+      "rewards/margins": 0.22561049461364746,
+      "rewards/rejected": 0.23164963722229004,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 2.8125,
+      "grad_norm": 13.25,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.63128662109375,
-      "logits/rejected": 83.65205383300781,
-      "logps/chosen": -33.91933822631836,
-      "logps/rejected": -31.633739471435547,
-      "loss": 0.4933,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.038565218448638916,
-      "rewards/margins": 0.026884615421295166,
-      "rewards/rejected": 0.01168060302734375,
+      "logits/chosen": 83.6882095336914,
+      "logits/rejected": 83.7122802734375,
+      "logps/chosen": -33.82537841796875,
+      "logps/rejected": -31.4283504486084,
+      "loss": 0.4705,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.38369041681289673,
+      "rewards/margins": 0.1259343922138214,
+      "rewards/rejected": 0.2577560245990753,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.67877960205078,
-      "eval_logits/rejected": 98.66271209716797,
-      "eval_logps/chosen": -32.3816032409668,
-      "eval_logps/rejected": -35.7989501953125,
-      "eval_loss": 0.5022624135017395,
-      "eval_rewards/accuracies": 0.4497508704662323,
-      "eval_rewards/chosen": 0.006157390773296356,
-      "eval_rewards/margins": -0.010600303299725056,
-      "eval_rewards/rejected": 0.016757693141698837,
-      "eval_runtime": 104.2733,
-      "eval_samples_per_second": 3.289,
+      "eval_logits/chosen": 98.68708801269531,
+      "eval_logits/rejected": 98.67501068115234,
+      "eval_logps/chosen": -32.27544021606445,
+      "eval_logps/rejected": -35.81333541870117,
+      "eval_loss": 0.4946621358394623,
+      "eval_rewards/accuracies": 0.4896179437637329,
+      "eval_rewards/chosen": 0.13418929278850555,
+      "eval_rewards/margins": 0.011633642949163914,
+      "eval_rewards/rejected": 0.12255565822124481,
+      "eval_runtime": 104.4117,
+      "eval_samples_per_second": 3.285,
       "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 2.8125,
+      "grad_norm": 22.75,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.84017944335938,
-      "logits/rejected": 83.70924377441406,
-      "logps/chosen": -32.02631759643555,
-      "logps/rejected": -32.62633514404297,
-      "loss": 0.478,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.09964846074581146,
-      "rewards/margins": 0.09669323265552521,
-      "rewards/rejected": 0.002955238102003932,
+      "logits/chosen": 83.8403091430664,
+      "logits/rejected": 83.71327209472656,
+      "logps/chosen": -31.912221908569336,
+      "logps/rejected": -32.58714294433594,
+      "loss": 0.3643,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.8884676694869995,
+      "rewards/margins": 0.8334689140319824,
+      "rewards/rejected": 0.054998766630887985,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 3.125,
+      "grad_norm": 22.5,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.92427062988281,
-      "logits/rejected": 84.0443115234375,
-      "logps/chosen": -27.96859359741211,
-      "logps/rejected": -35.113182067871094,
-      "loss": 0.4863,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.09609892219305038,
-      "rewards/margins": 0.06150230020284653,
-      "rewards/rejected": 0.03459661453962326,
+      "logits/chosen": 83.93721008300781,
+      "logits/rejected": 84.06002044677734,
+      "logps/chosen": -27.933292388916016,
+      "logps/rejected": -35.20619201660156,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.7970338463783264,
+      "rewards/margins": 0.594670295715332,
+      "rewards/rejected": 0.2023634910583496,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 2.125,
+      "grad_norm": 15.25,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 81.16522979736328,
-      "logits/rejected": 81.19355773925781,
-      "logps/chosen": -30.12435531616211,
-      "logps/rejected": -31.579242706298828,
-      "loss": 0.4897,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.07829995453357697,
-      "rewards/margins": 0.04306120425462723,
-      "rewards/rejected": 0.03523875027894974,
+      "logits/chosen": 81.2428970336914,
+      "logits/rejected": 81.28059387207031,
+      "logps/chosen": -30.123279571533203,
+      "logps/rejected": -31.811786651611328,
+      "loss": 0.4158,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.6272625923156738,
+      "rewards/margins": 0.5313904881477356,
+      "rewards/rejected": 0.09587211161851883,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 2.6875,
+      "grad_norm": 16.375,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.11293029785156,
-      "logits/rejected": 82.13672637939453,
-      "logps/chosen": -26.66647720336914,
-      "logps/rejected": -32.56136703491211,
-      "loss": 0.4804,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.07808488607406616,
-      "rewards/margins": 0.09188400208950043,
-      "rewards/rejected": -0.013799121603369713,
+      "logits/chosen": 82.30902862548828,
+      "logits/rejected": 82.33167266845703,
+      "logps/chosen": -26.606557846069336,
+      "logps/rejected": -32.55984115600586,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.6726120710372925,
+      "rewards/margins": 0.7817814946174622,
+      "rewards/rejected": -0.10916946828365326,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 2.765625,
+      "grad_norm": 12.9375,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.58255767822266,
-      "logits/rejected": 80.55290222167969,
-      "logps/chosen": -28.511905670166016,
-      "logps/rejected": -32.55813217163086,
-      "loss": 0.4806,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.087720587849617,
-      "rewards/margins": 0.08688360452651978,
-      "rewards/rejected": 0.0008369755814783275,
+      "logits/chosen": 80.79722595214844,
+      "logits/rejected": 80.76870727539062,
+      "logps/chosen": -28.53302001953125,
+      "logps/rejected": -32.609378814697266,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.6848722696304321,
+      "rewards/margins": 0.719173014163971,
+      "rewards/rejected": -0.03430069237947464,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 2.46875,
+      "grad_norm": 19.75,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 82.26856994628906,
-      "logits/rejected": 82.28471374511719,
-      "logps/chosen": -33.05678176879883,
-      "logps/rejected": -30.106842041015625,
-      "loss": 0.4733,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.10799698531627655,
-      "rewards/margins": 0.12342723459005356,
-      "rewards/rejected": -0.015430237166583538,
+      "logits/chosen": 82.5254135131836,
+      "logits/rejected": 82.54801940917969,
+      "logps/chosen": -33.080387115478516,
+      "logps/rejected": -30.11862564086914,
+      "loss": 0.3503,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.8450896143913269,
+      "rewards/margins": 0.9779588580131531,
+      "rewards/rejected": -0.13286921381950378,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 2.296875,
+      "grad_norm": 10.5,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 82.95423126220703,
-      "logits/rejected": 82.90636444091797,
-      "logps/chosen": -30.332260131835938,
-      "logps/rejected": -32.45390701293945,
-      "loss": 0.4717,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0927029699087143,
-      "rewards/margins": 0.13608744740486145,
-      "rewards/rejected": -0.04338447377085686,
+      "logits/chosen": 83.30370330810547,
+      "logits/rejected": 83.26591491699219,
+      "logps/chosen": -30.516061782836914,
+      "logps/rejected": -32.39433288574219,
+      "loss": 0.3746,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.5945821404457092,
+      "rewards/margins": 0.8940039873123169,
+      "rewards/rejected": -0.29942184686660767,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 1.953125,
+      "grad_norm": 7.34375,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 80.42130279541016,
-      "logits/rejected": 80.39375305175781,
-      "logps/chosen": -30.32647132873535,
-      "logps/rejected": -31.406139373779297,
-      "loss": 0.4766,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.08706829696893692,
-      "rewards/margins": 0.11196158826351166,
-      "rewards/rejected": -0.024893300607800484,
+      "logits/chosen": 80.87037658691406,
+      "logits/rejected": 80.84497833251953,
+      "logps/chosen": -30.32219886779785,
+      "logps/rejected": -31.39385414123535,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6999613642692566,
+      "rewards/margins": 0.8892775774002075,
+      "rewards/rejected": -0.18931610882282257,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 1.7421875,
+      "grad_norm": 9.25,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 82.06262969970703,
-      "logits/rejected": 82.0372543334961,
-      "logps/chosen": -29.780080795288086,
-      "logps/rejected": -30.145153045654297,
-      "loss": 0.4863,
+      "logits/chosen": 82.518310546875,
+      "logits/rejected": 82.48826599121094,
+      "logps/chosen": -29.71840476989746,
+      "logps/rejected": -30.17477798461914,
+      "loss": 0.4079,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0986335426568985,
-      "rewards/margins": 0.06634658575057983,
-      "rewards/rejected": 0.03228696063160896,
+      "rewards/chosen": 0.8384085893630981,
+      "rewards/margins": 0.6038115620613098,
+      "rewards/rejected": 0.23459705710411072,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 2.28125,
+      "grad_norm": 9.0625,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 77.48158264160156,
-      "logits/rejected": 77.42652893066406,
-      "logps/chosen": -32.95527267456055,
-      "logps/rejected": -32.44263458251953,
-      "loss": 0.4602,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.179746612906456,
-      "rewards/margins": 0.17466942965984344,
-      "rewards/rejected": 0.005077171605080366,
+      "logits/chosen": 77.99578094482422,
+      "logits/rejected": 77.94181060791016,
+      "logps/chosen": -32.918636322021484,
+      "logps/rejected": -32.197086334228516,
+      "loss": 0.3097,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.4672811031341553,
+      "rewards/margins": 1.2302258014678955,
+      "rewards/rejected": 0.23705501854419708,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.35516357421875,
-      "eval_logits/rejected": 98.3409652709961,
-      "eval_logps/chosen": -32.375885009765625,
-      "eval_logps/rejected": -36.08698654174805,
-      "eval_loss": 0.49672064185142517,
-      "eval_rewards/accuracies": 0.5510797500610352,
-      "eval_rewards/chosen": 0.006729110609740019,
-      "eval_rewards/margins": 0.01877453364431858,
-      "eval_rewards/rejected": -0.012045422568917274,
-      "eval_runtime": 104.099,
-      "eval_samples_per_second": 3.295,
+      "eval_logits/chosen": 98.47710418701172,
+      "eval_logits/rejected": 98.47020721435547,
+      "eval_logps/chosen": -32.242671966552734,
+      "eval_logps/rejected": -35.840518951416016,
+      "eval_loss": 0.49102693796157837,
+      "eval_rewards/accuracies": 0.5481727719306946,
+      "eval_rewards/chosen": 0.1604028344154358,
+      "eval_rewards/margins": 0.0595964714884758,
+      "eval_rewards/rejected": 0.1008063480257988,
+      "eval_runtime": 104.1775,
+      "eval_samples_per_second": 3.292,
       "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 3.8125,
+      "grad_norm": 20.25,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 80.07341003417969,
-      "logits/rejected": 79.99302673339844,
-      "logps/chosen": -32.486534118652344,
-      "logps/rejected": -34.771583557128906,
-      "loss": 0.4711,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.14591845870018005,
-      "rewards/margins": 0.12446693331003189,
-      "rewards/rejected": 0.021451527252793312,
+      "logits/chosen": 80.51226806640625,
+      "logits/rejected": 80.43040466308594,
+      "logps/chosen": -32.629119873046875,
+      "logps/rejected": -34.76416778564453,
+      "loss": 0.3655,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 1.053282618522644,
+      "rewards/margins": 0.87574303150177,
+      "rewards/rejected": 0.1775396168231964,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 2.875,
+      "grad_norm": 14.5,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 82.1479721069336,
-      "logits/rejected": 82.22358703613281,
-      "logps/chosen": -30.500757217407227,
-      "logps/rejected": -30.712026596069336,
-      "loss": 0.4675,
+      "logits/chosen": 82.59284973144531,
+      "logits/rejected": 82.66585540771484,
+      "logps/chosen": -30.357955932617188,
+      "logps/rejected": -30.737834930419922,
+      "loss": 0.2951,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.14481952786445618,
-      "rewards/margins": 0.14312690496444702,
-      "rewards/rejected": 0.0016926426906138659,
+      "rewards/chosen": 1.2727962732315063,
+      "rewards/margins": 1.2799025774002075,
+      "rewards/rejected": -0.007106387056410313,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 2.421875,
+      "grad_norm": 12.0625,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 79.23271179199219,
-      "logits/rejected": 79.28483581542969,
-      "logps/chosen": -31.72991371154785,
-      "logps/rejected": -33.9786376953125,
-      "loss": 0.4772,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.10758310556411743,
-      "rewards/margins": 0.1042320728302002,
-      "rewards/rejected": 0.00335102342069149,
+      "logits/chosen": 79.70103454589844,
+      "logits/rejected": 79.75657653808594,
+      "logps/chosen": -31.675893783569336,
+      "logps/rejected": -33.84308624267578,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.9038804173469543,
+      "rewards/margins": 0.7686265707015991,
+      "rewards/rejected": 0.1352538764476776,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 2.859375,
+      "grad_norm": 20.875,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 81.69049835205078,
-      "logits/rejected": 81.97315979003906,
-      "logps/chosen": -30.010046005249023,
-      "logps/rejected": -31.530466079711914,
-      "loss": 0.4621,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.16780194640159607,
-      "rewards/margins": 0.16764438152313232,
-      "rewards/rejected": 0.0001575589121785015,
+      "logits/chosen": 82.11561584472656,
+      "logits/rejected": 82.39993286132812,
+      "logps/chosen": -30.073062896728516,
+      "logps/rejected": -31.364795684814453,
+      "loss": 0.3068,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.2920019626617432,
+      "rewards/margins": 1.158202886581421,
+      "rewards/rejected": 0.13379916548728943,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 2.5625,
+      "grad_norm": 15.4375,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 80.38240814208984,
-      "logits/rejected": 80.4458999633789,
-      "logps/chosen": -26.418010711669922,
-      "logps/rejected": -29.8955020904541,
-      "loss": 0.4744,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.11778855323791504,
-      "rewards/margins": 0.11726468801498413,
-      "rewards/rejected": 0.000523855909705162,
+      "logits/chosen": 80.80778503417969,
+      "logits/rejected": 80.8788833618164,
+      "logps/chosen": -26.4283504486084,
+      "logps/rejected": -29.62093162536621,
+      "loss": 0.3934,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.934036135673523,
+      "rewards/margins": 0.7101906538009644,
+      "rewards/rejected": 0.22384557127952576,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 3.015625,
+      "grad_norm": 13.1875,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 77.50282287597656,
-      "logits/rejected": 77.68643951416016,
-      "logps/chosen": -29.69681739807129,
-      "logps/rejected": -36.06591033935547,
-      "loss": 0.4582,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1864176243543625,
-      "rewards/margins": 0.18284575641155243,
-      "rewards/rejected": 0.0035718693397939205,
+      "logits/chosen": 78.03572082519531,
+      "logits/rejected": 78.22270202636719,
+      "logps/chosen": -29.80512046813965,
+      "logps/rejected": -35.99930953979492,
+      "loss": 0.297,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.4046982526779175,
+      "rewards/margins": 1.3228428363800049,
+      "rewards/rejected": 0.08185521513223648,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 1.9453125,
+      "grad_norm": 9.3125,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 76.93331909179688,
-      "logits/rejected": 76.97276306152344,
-      "logps/chosen": -30.158321380615234,
-      "logps/rejected": -31.34457015991211,
-      "loss": 0.4668,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.16206035017967224,
-      "rewards/margins": 0.14597512781620026,
-      "rewards/rejected": 0.016085227951407433,
+      "logits/chosen": 77.41798400878906,
+      "logits/rejected": 77.4520034790039,
+      "logps/chosen": -30.092838287353516,
+      "logps/rejected": -31.44430160522461,
+      "loss": 0.305,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.3488702774047852,
+      "rewards/margins": 1.2999744415283203,
+      "rewards/rejected": 0.04889560490846634,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 3.53125,
+      "grad_norm": 20.125,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 79.60905456542969,
-      "logits/rejected": 79.37342071533203,
-      "logps/chosen": -30.423480987548828,
-      "logps/rejected": -29.19866943359375,
-      "loss": 0.4787,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.13864143192768097,
-      "rewards/margins": 0.09180860966444016,
-      "rewards/rejected": 0.04683280736207962,
+      "logits/chosen": 80.09739685058594,
+      "logits/rejected": 79.87271881103516,
+      "logps/chosen": -30.528423309326172,
+      "logps/rejected": -29.228124618530273,
+      "loss": 0.382,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 1.0251753330230713,
+      "rewards/margins": 0.6740784049034119,
+      "rewards/rejected": 0.3510969281196594,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 2.34375,
+      "grad_norm": 14.25,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 79.83843994140625,
-      "logits/rejected": 79.75175476074219,
-      "logps/chosen": -32.15901565551758,
-      "logps/rejected": -32.140541076660156,
-      "loss": 0.4529,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.1945231556892395,
-      "rewards/margins": 0.20689085125923157,
-      "rewards/rejected": -0.012367685325443745,
+      "logits/chosen": 80.29197692871094,
+      "logits/rejected": 80.20320129394531,
+      "logps/chosen": -32.255340576171875,
+      "logps/rejected": -32.0814208984375,
+      "loss": 0.2756,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.4791247844696045,
+      "rewards/margins": 1.530770182609558,
+      "rewards/rejected": -0.05164545774459839,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 2.0625,
+      "grad_norm": 14.625,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 75.40689849853516,
-      "logits/rejected": 75.46818542480469,
-      "logps/chosen": -31.496240615844727,
-      "logps/rejected": -28.952880859375,
-      "loss": 0.4586,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.19094672799110413,
-      "rewards/margins": 0.17603248357772827,
-      "rewards/rejected": 0.014914242550730705,
+      "logits/chosen": 75.86343383789062,
+      "logits/rejected": 75.9235610961914,
+      "logps/chosen": -31.682353973388672,
+      "logps/rejected": -28.961742401123047,
+      "loss": 0.3066,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 1.37868332862854,
+      "rewards/margins": 1.2664577960968018,
+      "rewards/rejected": 0.11222562938928604,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 98.27913665771484,
-      "eval_logits/rejected": 98.27122497558594,
-      "eval_logps/chosen": -32.257171630859375,
-      "eval_logps/rejected": -35.840736389160156,
-      "eval_loss": 0.4993577003479004,
-      "eval_rewards/accuracies": 0.5128737688064575,
-      "eval_rewards/chosen": 0.018600592389702797,
-      "eval_rewards/margins": 0.006022024899721146,
-      "eval_rewards/rejected": 0.012578567489981651,
-      "eval_runtime": 103.8539,
-      "eval_samples_per_second": 3.303,
-      "eval_steps_per_second": 0.414,
+      "eval_logits/chosen": 98.42933654785156,
+      "eval_logits/rejected": 98.42666625976562,
+      "eval_logps/chosen": -32.235782623291016,
+      "eval_logps/rejected": -35.84326934814453,
+      "eval_loss": 0.48553377389907837,
+      "eval_rewards/accuracies": 0.5282392501831055,
+      "eval_rewards/chosen": 0.16591256856918335,
+      "eval_rewards/margins": 0.06730752438306808,
+      "eval_rewards/rejected": 0.09860503673553467,
+      "eval_runtime": 104.1659,
+      "eval_samples_per_second": 3.293,
+      "eval_steps_per_second": 0.413,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 2.453125,
+      "grad_norm": 12.8125,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 82.53901672363281,
-      "logits/rejected": 82.57080078125,
-      "logps/chosen": -29.314640045166016,
-      "logps/rejected": -32.252933502197266,
-      "loss": 0.4642,
+      "logits/chosen": 82.97637176513672,
+      "logits/rejected": 83.01033020019531,
+      "logps/chosen": -29.268829345703125,
+      "logps/rejected": -32.1494026184082,
+      "loss": 0.3157,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.15488943457603455,
-      "rewards/margins": 0.16020672023296356,
-      "rewards/rejected": -0.005317269824445248,
+      "rewards/chosen": 1.2757642269134521,
+      "rewards/margins": 1.2354792356491089,
+      "rewards/rejected": 0.04028487950563431,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 2.515625,
+      "grad_norm": 15.4375,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 79.94550323486328,
-      "logits/rejected": 79.94490814208984,
-      "logps/chosen": -29.682098388671875,
-      "logps/rejected": -28.85439109802246,
-      "loss": 0.459,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.19164641201496124,
-      "rewards/margins": 0.18513616919517517,
-      "rewards/rejected": 0.006510235369205475,
+      "logits/chosen": 80.3984603881836,
+      "logits/rejected": 80.40061950683594,
+      "logps/chosen": -29.979394912719727,
+      "logps/rejected": -28.756725311279297,
+      "loss": 0.3119,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.2953336238861084,
+      "rewards/margins": 1.1651219129562378,
+      "rewards/rejected": 0.13021180033683777,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 3.09375,
+      "grad_norm": 10.75,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 77.13621520996094,
-      "logits/rejected": 77.16484069824219,
-      "logps/chosen": -28.32808494567871,
-      "logps/rejected": -32.60577392578125,
-      "loss": 0.4534,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.20245198905467987,
-      "rewards/margins": 0.19954310357570648,
-      "rewards/rejected": 0.002908907597884536,
+      "logits/chosen": 77.65103912353516,
+      "logits/rejected": 77.68055725097656,
+      "logps/chosen": -28.304912567138672,
+      "logps/rejected": -32.50617218017578,
+      "loss": 0.254,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.6381542682647705,
+      "rewards/margins": 1.535199522972107,
+      "rewards/rejected": 0.10295436531305313,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 3.046875,
+      "grad_norm": 15.9375,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 81.41078186035156,
-      "logits/rejected": 81.44329071044922,
-      "logps/chosen": -31.582672119140625,
-      "logps/rejected": -33.35100555419922,
-      "loss": 0.4655,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.16294743120670319,
-      "rewards/margins": 0.16117897629737854,
-      "rewards/rejected": 0.0017684675985947251,
+      "logits/chosen": 81.86819458007812,
+      "logits/rejected": 81.89158630371094,
+      "logps/chosen": -31.62424659729004,
+      "logps/rejected": -33.315486907958984,
+      "loss": 0.3285,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.2703235149383545,
+      "rewards/margins": 1.2277603149414062,
+      "rewards/rejected": 0.0425633080303669,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 2.34375,
+      "grad_norm": 9.3125,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 80.47315216064453,
-      "logits/rejected": 80.48896789550781,
-      "logps/chosen": -31.6666316986084,
-      "logps/rejected": -32.75847625732422,
-      "loss": 0.4641,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.1977105438709259,
-      "rewards/margins": 0.1643778532743454,
-      "rewards/rejected": 0.03333265334367752,
+      "logits/chosen": 80.94864654541016,
+      "logits/rejected": 80.96649169921875,
+      "logps/chosen": -31.77408218383789,
+      "logps/rejected": -32.98318099975586,
+      "loss": 0.3071,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 1.4957287311553955,
+      "rewards/margins": 1.4088352918624878,
+      "rewards/rejected": 0.08689363300800323,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 2.5625,
+      "grad_norm": 11.75,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 81.98124694824219,
-      "logits/rejected": 82.04177856445312,
-      "logps/chosen": -27.832752227783203,
-      "logps/rejected": -31.519290924072266,
-      "loss": 0.4636,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17194589972496033,
-      "rewards/margins": 0.15889012813568115,
-      "rewards/rejected": 0.013055793941020966,
+      "logits/chosen": 82.43021392822266,
+      "logits/rejected": 82.48263549804688,
+      "logps/chosen": -27.799968719482422,
+      "logps/rejected": -31.40801429748535,
+      "loss": 0.3137,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.4017945528030396,
+      "rewards/margins": 1.2083251476287842,
+      "rewards/rejected": 0.19346928596496582,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 2.609375,
+      "grad_norm": 12.1875,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 81.46924591064453,
-      "logits/rejected": 81.51275634765625,
-      "logps/chosen": -31.354238510131836,
-      "logps/rejected": -34.80229568481445,
-      "loss": 0.4735,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.15219244360923767,
-      "rewards/margins": 0.1214846819639206,
-      "rewards/rejected": 0.03070778027176857,
+      "logits/chosen": 81.93843078613281,
+      "logits/rejected": 81.97197723388672,
+      "logps/chosen": -31.361141204833984,
+      "logps/rejected": -34.82567596435547,
+      "loss": 0.3594,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 1.2120168209075928,
+      "rewards/margins": 0.9850583076477051,
+      "rewards/rejected": 0.22695855796337128,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 2.546875,
+      "grad_norm": 14.9375,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 75.28022766113281,
-      "logits/rejected": 75.15403747558594,
-      "logps/chosen": -28.978900909423828,
-      "logps/rejected": -27.919509887695312,
-      "loss": 0.4691,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.15972287952899933,
-      "rewards/margins": 0.13561637699604034,
-      "rewards/rejected": 0.024106483906507492,
+      "logits/chosen": 75.7864761352539,
+      "logits/rejected": 75.65766143798828,
+      "logps/chosen": -29.02054786682129,
+      "logps/rejected": -27.994884490966797,
+      "loss": 0.3268,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 1.244463562965393,
+      "rewards/margins": 1.111914038658142,
+      "rewards/rejected": 0.13254934549331665,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.47652822160101554,
-      "train_runtime": 2558.0485,
-      "train_samples_per_second": 1.204,
-      "train_steps_per_second": 0.151
+      "train_loss": 0.37664644687206716,
+      "train_runtime": 2560.444,
+      "train_samples_per_second": 1.203,
+      "train_steps_per_second": 0.15
     }
   ],
   "logging_steps": 10,