Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4955
-- Rewards/chosen: 0.0848
-- Rewards/rejected: 0.0593
-- Rewards/accuracies: 0.5282
-- Rewards/margins: 0.0255
-- Logps/rejected: -35.8183
-- Logps/chosen: -32.2312
-- Logits/rejected: 98.3492
-- Logits/chosen: 98.3463
+- Loss: 0.4994
+- Rewards/chosen: 0.0186
+- Rewards/rejected: 0.0126
+- Rewards/accuracies: 0.5129
+- Rewards/margins: 0.0060
+- Logps/rejected: -35.8407
+- Logps/chosen: -32.2572
+- Logits/rejected: 98.2712
+- Logits/chosen: 98.2791
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4749        | 0.26  | 100  | 0.5047          | 0.0131         | 0.0430           | 0.4747             | -0.0299         | -35.8591       | -32.4105     | 98.7377         | 98.7500       |
-| 0.3843        | 0.52  | 200  | 0.4944          | 0.0238         | -0.0061          | 0.5307             | 0.0299          | -35.9817       | -32.3837     | 98.3784         | 98.3836       |
-| 0.363         | 0.78  | 300  | 0.4962          | 0.0856         | 0.0600           | 0.5104             | 0.0256          | -35.8166       | -32.2293     | 98.3704         | 98.3655       |
+| 0.4933        | 0.26  | 100  | 0.5023          | 0.0062         | 0.0168           | 0.4498             | -0.0106         | -35.7990       | -32.3816     | 98.6627         | 98.6788       |
+| 0.4602        | 0.52  | 200  | 0.4967          | 0.0067         | -0.0120          | 0.5511             | 0.0188          | -36.0870       | -32.3759     | 98.3410         | 98.3552       |
+| 0.4586        | 0.78  | 300  | 0.4994          | 0.0186         | 0.0126           | 0.5129             | 0.0060          | -35.8407       | -32.2572     | 98.2712         | 98.2791       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f04c43e5ec9b3dc2d9cc94df254a185cd9d8abaab456b54b90ea0a9efcbcfdb9
+oid sha256:c8168ae0fb287b1ef2742e5807dc5ef63898668f13fa8511336f2612c339c708
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.34632873535156,
-    "eval_logits/rejected": 98.3492202758789,
-    "eval_logps/chosen": -32.23118591308594,
-    "eval_logps/rejected": -35.818275451660156,
-    "eval_loss": 0.4955456852912903,
-    "eval_rewards/accuracies": 0.5282392501831055,
-    "eval_rewards/chosen": 0.08479735255241394,
-    "eval_rewards/margins": 0.025497542694211006,
-    "eval_rewards/rejected": 0.059299811720848083,
-    "eval_runtime": 103.8037,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.304,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.4199758805237807,
-    "train_runtime": 2559.5376,
+    "train_loss": 0.47652822160101554,
+    "train_runtime": 2558.0485,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.204,
+    "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4199758805237807,
-    "train_runtime": 2559.5376,
+    "train_loss": 0.47652822160101554,
+    "train_runtime": 2558.0485,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.204,
+    "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 13.0625,
+      "grad_norm": 3.265625,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 5.375,
+      "grad_norm": 1.625,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.08451080322266,
-      "logits/rejected": 80.78488159179688,
-      "logps/chosen": -34.28126525878906,
-      "logps/rejected": -33.13351058959961,
-      "loss": 0.492,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": -0.015815330669283867,
-      "rewards/margins": 0.05422591418027878,
-      "rewards/rejected": -0.0700412467122078,
+      "logits/chosen": 81.08853912353516,
+      "logits/rejected": 80.79112243652344,
+      "logps/chosen": -34.252471923828125,
+      "logps/rejected": -32.9941291809082,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": -0.0010747427586466074,
+      "rewards/margins": 0.0024977123830467463,
+      "rewards/rejected": -0.003572455607354641,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 9.6875,
+      "grad_norm": 2.546875,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.63957214355469,
-      "logits/rejected": 80.53150939941406,
-      "logps/chosen": -33.59695816040039,
-      "logps/rejected": -30.76776123046875,
-      "loss": 0.4944,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.03718096390366554,
-      "rewards/margins": 0.03328876942396164,
-      "rewards/rejected": 0.003892195178195834,
+      "logits/chosen": 80.66322326660156,
+      "logits/rejected": 80.55162048339844,
+      "logps/chosen": -33.64655303955078,
+      "logps/rejected": -30.839855194091797,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004335404373705387,
+      "rewards/margins": 0.010572222992777824,
+      "rewards/rejected": -0.006236819084733725,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 10.5,
+      "grad_norm": 2.140625,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.47281646728516,
-      "logits/rejected": 82.50285339355469,
-      "logps/chosen": -33.86055374145508,
-      "logps/rejected": -31.097183227539062,
-      "loss": 0.5134,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 0.06090690940618515,
-      "rewards/margins": -0.0697535052895546,
-      "rewards/rejected": 0.13066044449806213,
+      "logits/chosen": 82.4841079711914,
+      "logits/rejected": 82.51509094238281,
+      "logps/chosen": -33.830867767333984,
+      "logps/rejected": -31.266870498657227,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.018195267766714096,
+      "rewards/margins": 0.002499166876077652,
+      "rewards/rejected": 0.015696100890636444,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 10.875,
+      "grad_norm": 2.90625,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.05327606201172,
-      "logits/rejected": 81.04902648925781,
-      "logps/chosen": -32.787689208984375,
-      "logps/rejected": -33.05846405029297,
-      "loss": 0.4913,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.15454962849617004,
-      "rewards/margins": 0.030526524409651756,
-      "rewards/rejected": 0.12402307987213135,
+      "logits/chosen": 81.04556274414062,
+      "logits/rejected": 81.04271697998047,
+      "logps/chosen": -32.56133270263672,
+      "logps/rejected": -33.02995300292969,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.06127279996871948,
+      "rewards/margins": 0.0274154394865036,
+      "rewards/rejected": 0.03385736048221588,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 8.125,
+      "grad_norm": 2.84375,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.79315185546875,
-      "logits/rejected": 78.80322265625,
-      "logps/chosen": -30.44384765625,
-      "logps/rejected": -30.80826759338379,
-      "loss": 0.4599,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.30455905199050903,
-      "rewards/margins": 0.19855086505413055,
-      "rewards/rejected": 0.10600819438695908,
+      "logits/chosen": 78.80296325683594,
+      "logits/rejected": 78.81227111816406,
+      "logps/chosen": -30.36850929260254,
+      "logps/rejected": -30.71665382385254,
+      "loss": 0.4889,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08367374539375305,
+      "rewards/margins": 0.04801047220826149,
+      "rewards/rejected": 0.03566327691078186,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 9.625,
+      "grad_norm": 1.8984375,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.44609069824219,
-      "logits/rejected": 83.50300598144531,
-      "logps/chosen": -30.81852149963379,
-      "logps/rejected": -29.231618881225586,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 0.1426558792591095,
-      "rewards/margins": -0.0009051367524079978,
-      "rewards/rejected": 0.1435610055923462,
+      "logits/chosen": 83.3783187866211,
+      "logits/rejected": 83.4344482421875,
+      "logps/chosen": -30.705408096313477,
+      "logps/rejected": -29.251195907592773,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0469752736389637,
+      "rewards/margins": 0.013043051585555077,
+      "rewards/rejected": 0.033932216465473175,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 7.875,
+      "grad_norm": 2.828125,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 84.02252960205078,
-      "logits/rejected": 84.05018615722656,
-      "logps/chosen": -30.281408309936523,
-      "logps/rejected": -32.76702880859375,
-      "loss": 0.4981,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.13904651999473572,
-      "rewards/margins": 0.03077618218958378,
-      "rewards/rejected": 0.10827036201953888,
+      "logits/chosen": 83.97593688964844,
+      "logits/rejected": 84.0093765258789,
+      "logps/chosen": -30.22530746459961,
+      "logps/rejected": -32.46759033203125,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.0403718426823616,
+      "rewards/margins": -0.016639459878206253,
+      "rewards/rejected": 0.05701129883527756,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 9.5,
+      "grad_norm": 2.78125,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.84428405761719,
-      "logits/rejected": 81.82522583007812,
-      "logps/chosen": -31.032154083251953,
-      "logps/rejected": -30.712255477905273,
-      "loss": 0.4648,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.18911631405353546,
-      "rewards/margins": 0.1709602177143097,
-      "rewards/rejected": 0.018156107515096664,
+      "logits/chosen": 81.67536163330078,
+      "logits/rejected": 81.65579986572266,
+      "logps/chosen": -31.098180770874023,
+      "logps/rejected": -30.70516014099121,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.040676504373550415,
+      "rewards/margins": 0.035427749156951904,
+      "rewards/rejected": 0.005248754285275936,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 12.375,
+      "grad_norm": 3.375,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.68492126464844,
-      "logits/rejected": 78.66017150878906,
-      "logps/chosen": -32.12371063232422,
-      "logps/rejected": -30.837940216064453,
-      "loss": 0.4704,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.20578794181346893,
-      "rewards/margins": 0.16037426888942719,
-      "rewards/rejected": 0.04541371017694473,
+      "logits/chosen": 78.46135711669922,
+      "logits/rejected": 78.43350982666016,
+      "logps/chosen": -32.169097900390625,
+      "logps/rejected": -30.87672996520996,
+      "loss": 0.4912,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04690824821591377,
+      "rewards/margins": 0.03943391516804695,
+      "rewards/rejected": 0.007474330719560385,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 9.9375,
+      "grad_norm": 2.8125,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.77175903320312,
-      "logits/rejected": 83.79080963134766,
-      "logps/chosen": -33.829063415527344,
-      "logps/rejected": -31.544397354125977,
-      "loss": 0.4749,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.1903691589832306,
-      "rewards/margins": 0.10790882259607315,
-      "rewards/rejected": 0.08246034383773804,
+      "logits/chosen": 83.63128662109375,
+      "logits/rejected": 83.65205383300781,
+      "logps/chosen": -33.91933822631836,
+      "logps/rejected": -31.633739471435547,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.038565218448638916,
+      "rewards/margins": 0.026884615421295166,
+      "rewards/rejected": 0.01168060302734375,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.75000762939453,
-      "eval_logits/rejected": 98.7376708984375,
-      "eval_logps/chosen": -32.41049575805664,
-      "eval_logps/rejected": -35.85907745361328,
-      "eval_loss": 0.504712700843811,
-      "eval_rewards/accuracies": 0.4746677577495575,
-      "eval_rewards/chosen": 0.013071590103209019,
-      "eval_rewards/margins": -0.02990747056901455,
-      "eval_rewards/rejected": 0.042979057878255844,
-      "eval_runtime": 104.3402,
-      "eval_samples_per_second": 3.287,
+      "eval_logits/chosen": 98.67877960205078,
+      "eval_logits/rejected": 98.66271209716797,
+      "eval_logps/chosen": -32.3816032409668,
+      "eval_logps/rejected": -35.7989501953125,
+      "eval_loss": 0.5022624135017395,
+      "eval_rewards/accuracies": 0.4497508704662323,
+      "eval_rewards/chosen": 0.006157390773296356,
+      "eval_rewards/margins": -0.010600303299725056,
+      "eval_rewards/rejected": 0.016757693141698837,
+      "eval_runtime": 104.2733,
+      "eval_samples_per_second": 3.289,
       "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 11.875,
+      "grad_norm": 2.8125,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 84.02198028564453,
-      "logits/rejected": 83.90216064453125,
-      "logps/chosen": -32.067047119140625,
-      "logps/rejected": -32.60837936401367,
-      "loss": 0.4251,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.382302463054657,
-      "rewards/margins": 0.36329856514930725,
-      "rewards/rejected": 0.01900387369096279,
+      "logits/chosen": 83.84017944335938,
+      "logits/rejected": 83.70924377441406,
+      "logps/chosen": -32.02631759643555,
+      "logps/rejected": -32.62633514404297,
+      "loss": 0.478,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.09964846074581146,
+      "rewards/margins": 0.09669323265552521,
+      "rewards/rejected": 0.002955238102003932,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 10.9375,
+      "grad_norm": 3.125,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 84.06869506835938,
-      "logits/rejected": 84.17839050292969,
-      "logps/chosen": -28.029781341552734,
-      "logps/rejected": -35.15131378173828,
-      "loss": 0.4499,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.35991987586021423,
-      "rewards/margins": 0.23678629100322723,
-      "rewards/rejected": 0.123133584856987,
+      "logits/chosen": 83.92427062988281,
+      "logits/rejected": 84.0443115234375,
+      "logps/chosen": -27.96859359741211,
+      "logps/rejected": -35.113182067871094,
+      "loss": 0.4863,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09609892219305038,
+      "rewards/margins": 0.06150230020284653,
+      "rewards/rejected": 0.03459661453962326,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 9.125,
+      "grad_norm": 2.125,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 81.26686096191406,
-      "logits/rejected": 81.29707336425781,
-      "logps/chosen": -30.0821533203125,
-      "logps/rejected": -31.722143173217773,
-      "loss": 0.4515,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.3300797641277313,
-      "rewards/margins": 0.24628591537475586,
-      "rewards/rejected": 0.08379384875297546,
+      "logits/chosen": 81.16522979736328,
+      "logits/rejected": 81.19355773925781,
+      "logps/chosen": -30.12435531616211,
+      "logps/rejected": -31.579242706298828,
+      "loss": 0.4897,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.07829995453357697,
+      "rewards/margins": 0.04306120425462723,
+      "rewards/rejected": 0.03523875027894974,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 7.09375,
+      "grad_norm": 2.6875,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.26895904541016,
-      "logits/rejected": 82.28730773925781,
-      "logps/chosen": -26.675317764282227,
-      "logps/rejected": -32.535396575927734,
-      "loss": 0.4328,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.30880242586135864,
-      "rewards/margins": 0.35360854864120483,
-      "rewards/rejected": -0.044806141406297684,
+      "logits/chosen": 82.11293029785156,
+      "logits/rejected": 82.13672637939453,
+      "logps/chosen": -26.66647720336914,
+      "logps/rejected": -32.56136703491211,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.07808488607406616,
+      "rewards/margins": 0.09188400208950043,
+      "rewards/rejected": -0.013799121603369713,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 8.75,
+      "grad_norm": 2.765625,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.73310089111328,
-      "logits/rejected": 80.70652770996094,
-      "logps/chosen": -28.468103408813477,
-      "logps/rejected": -32.77440643310547,
-      "loss": 0.4125,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.36840274930000305,
-      "rewards/margins": 0.45156532526016235,
-      "rewards/rejected": -0.08316260576248169,
+      "logits/chosen": 80.58255767822266,
+      "logits/rejected": 80.55290222167969,
+      "logps/chosen": -28.511905670166016,
+      "logps/rejected": -32.55813217163086,
+      "loss": 0.4806,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.087720587849617,
+      "rewards/margins": 0.08688360452651978,
+      "rewards/rejected": 0.0008369755814783275,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 8.5,
+      "grad_norm": 2.46875,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 82.41865539550781,
-      "logits/rejected": 82.44725799560547,
-      "logps/chosen": -33.112274169921875,
-      "logps/rejected": -30.1284122467041,
-      "loss": 0.4055,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.40979090332984924,
-      "rewards/margins": 0.4801415503025055,
-      "rewards/rejected": -0.07035063952207565,
+      "logits/chosen": 82.26856994628906,
+      "logits/rejected": 82.28471374511719,
+      "logps/chosen": -33.05678176879883,
+      "logps/rejected": -30.106842041015625,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.10799698531627655,
+      "rewards/margins": 0.12342723459005356,
+      "rewards/rejected": -0.015430237166583538,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 7.5625,
+      "grad_norm": 2.296875,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 83.09947204589844,
-      "logits/rejected": 83.05732727050781,
-      "logps/chosen": -30.413660049438477,
-      "logps/rejected": -32.43395233154297,
-      "loss": 0.4144,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.33825185894966125,
-      "rewards/margins": 0.5038093328475952,
-      "rewards/rejected": -0.16555748879909515,
+      "logits/chosen": 82.95423126220703,
+      "logits/rejected": 82.90636444091797,
+      "logps/chosen": -30.332260131835938,
+      "logps/rejected": -32.45390701293945,
+      "loss": 0.4717,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0927029699087143,
+      "rewards/margins": 0.13608744740486145,
+      "rewards/rejected": -0.04338447377085686,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 6.46875,
+      "grad_norm": 1.953125,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 80.59300231933594,
-      "logits/rejected": 80.56774139404297,
-      "logps/chosen": -30.52907371520996,
-      "logps/rejected": -31.434350967407227,
-      "loss": 0.433,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2672317624092102,
-      "rewards/margins": 0.3780880868434906,
-      "rewards/rejected": -0.1108563169836998,
+      "logits/chosen": 80.42130279541016,
+      "logits/rejected": 80.39375305175781,
+      "logps/chosen": -30.32647132873535,
+      "logps/rejected": -31.406139373779297,
+      "loss": 0.4766,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.08706829696893692,
+      "rewards/margins": 0.11196158826351166,
+      "rewards/rejected": -0.024893300607800484,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 5.40625,
+      "grad_norm": 1.7421875,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 82.23339080810547,
-      "logits/rejected": 82.1915283203125,
-      "logps/chosen": -29.8317813873291,
-      "logps/rejected": -30.190067291259766,
-      "loss": 0.4509,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.37385472655296326,
-      "rewards/margins": 0.26267164945602417,
-      "rewards/rejected": 0.1111830323934555,
+      "logits/chosen": 82.06262969970703,
+      "logits/rejected": 82.0372543334961,
+      "logps/chosen": -29.780080795288086,
+      "logps/rejected": -30.145153045654297,
+      "loss": 0.4863,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0986335426568985,
+      "rewards/margins": 0.06634658575057983,
+      "rewards/rejected": 0.03228696063160896,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 6.625,
+      "grad_norm": 2.28125,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 77.5392074584961,
-      "logits/rejected": 77.48648834228516,
-      "logps/chosen": -33.0440788269043,
-      "logps/rejected": -32.247684478759766,
-      "loss": 0.3843,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.6834636926651001,
-      "rewards/margins": 0.5851765871047974,
-      "rewards/rejected": 0.09828709810972214,
+      "logits/chosen": 77.48158264160156,
+      "logits/rejected": 77.42652893066406,
+      "logps/chosen": -32.95527267456055,
+      "logps/rejected": -32.44263458251953,
+      "loss": 0.4602,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.179746612906456,
+      "rewards/margins": 0.17466942965984344,
+      "rewards/rejected": 0.005077171605080366,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.38356018066406,
-      "eval_logits/rejected": 98.37836456298828,
-      "eval_logps/chosen": -32.3836784362793,
-      "eval_logps/rejected": -35.98166275024414,
-      "eval_loss": 0.4944417476654053,
-      "eval_rewards/accuracies": 0.5307309031486511,
-      "eval_rewards/chosen": 0.023798126727342606,
-      "eval_rewards/margins": 0.029852891340851784,
-      "eval_rewards/rejected": -0.00605476601049304,
-      "eval_runtime": 104.0926,
+      "eval_logits/chosen": 98.35516357421875,
+      "eval_logits/rejected": 98.3409652709961,
+      "eval_logps/chosen": -32.375885009765625,
+      "eval_logps/rejected": -36.08698654174805,
+      "eval_loss": 0.49672064185142517,
+      "eval_rewards/accuracies": 0.5510797500610352,
+      "eval_rewards/chosen": 0.006729110609740019,
+      "eval_rewards/margins": 0.01877453364431858,
+      "eval_rewards/rejected": -0.012045422568917274,
+      "eval_runtime": 104.099,
       "eval_samples_per_second": 3.295,
       "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 12.3125,
+      "grad_norm": 3.8125,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 80.09881591796875,
-      "logits/rejected": 80.01982879638672,
-      "logps/chosen": -32.544151306152344,
-      "logps/rejected": -35.001224517822266,
-      "loss": 0.3882,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.5606266260147095,
-      "rewards/margins": 0.5666781663894653,
-      "rewards/rejected": -0.006051521748304367,
+      "logits/chosen": 80.07341003417969,
+      "logits/rejected": 79.99302673339844,
+      "logps/chosen": -32.486534118652344,
+      "logps/rejected": -34.771583557128906,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.14591845870018005,
+      "rewards/margins": 0.12446693331003189,
+      "rewards/rejected": 0.021451527252793312,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 8.1875,
+      "grad_norm": 2.875,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 82.21588134765625,
-      "logits/rejected": 82.28758239746094,
-      "logps/chosen": -30.38724136352539,
-      "logps/rejected": -30.862102508544922,
-      "loss": 0.3707,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.6246866583824158,
-      "rewards/margins": 0.6779459714889526,
-      "rewards/rejected": -0.05325937271118164,
+      "logits/chosen": 82.1479721069336,
+      "logits/rejected": 82.22358703613281,
+      "logps/chosen": -30.500757217407227,
+      "logps/rejected": -30.712026596069336,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14481952786445618,
+      "rewards/margins": 0.14312690496444702,
+      "rewards/rejected": 0.0016926426906138659,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 8.5,
+      "grad_norm": 2.421875,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 79.33607482910156,
-      "logits/rejected": 79.39179992675781,
-      "logps/chosen": -31.715646743774414,
-      "logps/rejected": -33.92882537841797,
-      "loss": 0.4251,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4360387325286865,
-      "rewards/margins": 0.40270787477493286,
-      "rewards/rejected": 0.03333085775375366,
+      "logits/chosen": 79.23271179199219,
+      "logits/rejected": 79.28483581542969,
+      "logps/chosen": -31.72991371154785,
+      "logps/rejected": -33.9786376953125,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10758310556411743,
+      "rewards/margins": 0.1042320728302002,
+      "rewards/rejected": 0.00335102342069149,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 10.625,
+      "grad_norm": 2.859375,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 81.82881164550781,
-      "logits/rejected": 82.10590362548828,
-      "logps/chosen": -29.962848663330078,
-      "logps/rejected": -31.462871551513672,
-      "loss": 0.3667,
+      "logits/chosen": 81.69049835205078,
+      "logits/rejected": 81.97315979003906,
+      "logps/chosen": -30.010046005249023,
+      "logps/rejected": -31.530466079711914,
+      "loss": 0.4621,
       "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.6900869607925415,
-      "rewards/margins": 0.6624192595481873,
-      "rewards/rejected": 0.027667587623000145,
+      "rewards/chosen": 0.16780194640159607,
+      "rewards/margins": 0.16764438152313232,
+      "rewards/rejected": 0.0001575589121785015,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 11.25,
+      "grad_norm": 2.5625,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 80.50505065917969,
-      "logits/rejected": 80.56990051269531,
-      "logps/chosen": -26.388418197631836,
-      "logps/rejected": -29.717565536499023,
-      "loss": 0.4245,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.48299089074134827,
-      "rewards/margins": 0.40972191095352173,
-      "rewards/rejected": 0.07326899468898773,
+      "logits/chosen": 80.38240814208984,
+      "logits/rejected": 80.4458999633789,
+      "logps/chosen": -26.418010711669922,
+      "logps/rejected": -29.8955020904541,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.11778855323791504,
+      "rewards/margins": 0.11726468801498413,
+      "rewards/rejected": 0.000523855909705162,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 8.625,
+      "grad_norm": 3.015625,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 77.76419830322266,
-      "logits/rejected": 77.94038391113281,
-      "logps/chosen": -29.758447647094727,
-      "logps/rejected": -36.150657653808594,
-      "loss": 0.3604,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7210196256637573,
-      "rewards/margins": 0.7406320571899414,
-      "rewards/rejected": -0.0196123905479908,
+      "logits/chosen": 77.50282287597656,
+      "logits/rejected": 77.68643951416016,
+      "logps/chosen": -29.69681739807129,
+      "logps/rejected": -36.06591033935547,
+      "loss": 0.4582,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1864176243543625,
+      "rewards/margins": 0.18284575641155243,
+      "rewards/rejected": 0.0035718693397939205,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 6.1875,
+      "grad_norm": 1.9453125,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 77.1490707397461,
-      "logits/rejected": 77.18915557861328,
-      "logps/chosen": -30.186153411865234,
-      "logps/rejected": -31.373126983642578,
-      "loss": 0.3871,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.6371081471443176,
-      "rewards/margins": 0.5841903686523438,
-      "rewards/rejected": 0.05291769653558731,
+      "logits/chosen": 76.93331909179688,
+      "logits/rejected": 76.97276306152344,
+      "logps/chosen": -30.158321380615234,
+      "logps/rejected": -31.34457015991211,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.16206035017967224,
+      "rewards/margins": 0.14597512781620026,
+      "rewards/rejected": 0.016085227951407433,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 12.5,
+      "grad_norm": 3.53125,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 79.78591918945312,
-      "logits/rejected": 79.5605239868164,
-      "logps/chosen": -30.330188751220703,
-      "logps/rejected": -29.223669052124023,
-      "loss": 0.4182,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.5918818712234497,
-      "rewards/margins": 0.41455134749412537,
-      "rewards/rejected": 0.17733044922351837,
+      "logits/chosen": 79.60905456542969,
+      "logits/rejected": 79.37342071533203,
+      "logps/chosen": -30.423480987548828,
+      "logps/rejected": -29.19866943359375,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.13864143192768097,
+      "rewards/margins": 0.09180860966444016,
+      "rewards/rejected": 0.04683280736207962,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 8.4375,
+      "grad_norm": 2.34375,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 80.03771209716797,
-      "logits/rejected": 79.94876861572266,
-      "logps/chosen": -32.170814514160156,
-      "logps/rejected": -32.12440872192383,
-      "loss": 0.3449,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.7733729481697083,
-      "rewards/margins": 0.8163889646530151,
-      "rewards/rejected": -0.04301605746150017,
+      "logits/chosen": 79.83843994140625,
+      "logits/rejected": 79.75175476074219,
+      "logps/chosen": -32.15901565551758,
+      "logps/rejected": -32.140541076660156,
+      "loss": 0.4529,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.1945231556892395,
+      "rewards/margins": 0.20689085125923157,
+      "rewards/rejected": -0.012367685325443745,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 5.5,
+      "grad_norm": 2.0625,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 75.5302505493164,
-      "logits/rejected": 75.59835052490234,
-      "logps/chosen": -31.593181610107422,
-      "logps/rejected": -29.076126098632812,
-      "loss": 0.363,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.7250088453292847,
-      "rewards/margins": 0.714650571346283,
-      "rewards/rejected": 0.010358264669775963,
+      "logits/chosen": 75.40689849853516,
+      "logits/rejected": 75.46818542480469,
+      "logps/chosen": -31.496240615844727,
+      "logps/rejected": -28.952880859375,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.19094672799110413,
+      "rewards/margins": 0.17603248357772827,
+      "rewards/rejected": 0.014914242550730705,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 98.36553192138672,
-      "eval_logits/rejected": 98.37035369873047,
-      "eval_logps/chosen": -32.229251861572266,
-      "eval_logps/rejected": -35.81660079956055,
-      "eval_loss": 0.49616533517837524,
-      "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.0855708047747612,
-      "eval_rewards/margins": 0.025601176545023918,
-      "eval_rewards/rejected": 0.05996962636709213,
-      "eval_runtime": 104.1169,
-      "eval_samples_per_second": 3.294,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.27913665771484,
+      "eval_logits/rejected": 98.27122497558594,
+      "eval_logps/chosen": -32.257171630859375,
+      "eval_logps/rejected": -35.840736389160156,
+      "eval_loss": 0.4993577003479004,
+      "eval_rewards/accuracies": 0.5128737688064575,
+      "eval_rewards/chosen": 0.018600592389702797,
+      "eval_rewards/margins": 0.006022024899721146,
+      "eval_rewards/rejected": 0.012578567489981651,
+      "eval_runtime": 103.8539,
+      "eval_samples_per_second": 3.303,
+      "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 8.0625,
+      "grad_norm": 2.453125,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 82.7135238647461,
-      "logits/rejected": 82.74400329589844,
-      "logps/chosen": -29.19759178161621,
-      "logps/rejected": -32.07235336303711,
-      "loss": 0.3811,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.666375994682312,
-      "rewards/margins": 0.6154125332832336,
-      "rewards/rejected": 0.05096355825662613,
+      "logits/chosen": 82.53901672363281,
+      "logits/rejected": 82.57080078125,
+      "logps/chosen": -29.314640045166016,
+      "logps/rejected": -32.252933502197266,
+      "loss": 0.4642,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.15488943457603455,
+      "rewards/margins": 0.16020672023296356,
+      "rewards/rejected": -0.005317269824445248,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 8.5,
+      "grad_norm": 2.515625,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 80.10868835449219,
-      "logits/rejected": 80.10973358154297,
-      "logps/chosen": -29.693145751953125,
-      "logps/rejected": -28.737625122070312,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7621666193008423,
-      "rewards/margins": 0.6894195675849915,
-      "rewards/rejected": 0.07274699211120605,
+      "logits/chosen": 79.94550323486328,
+      "logits/rejected": 79.94490814208984,
+      "logps/chosen": -29.682098388671875,
+      "logps/rejected": -28.85439109802246,
+      "loss": 0.459,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.19164641201496124,
+      "rewards/margins": 0.18513616919517517,
+      "rewards/rejected": 0.006510235369205475,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 7.8125,
+      "grad_norm": 3.09375,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 77.36323547363281,
-      "logits/rejected": 77.391357421875,
-      "logps/chosen": -28.293853759765625,
-      "logps/rejected": -32.6038703918457,
-      "loss": 0.3393,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.8235009908676147,
-      "rewards/margins": 0.8111019134521484,
-      "rewards/rejected": 0.012399068102240562,
+      "logits/chosen": 77.13621520996094,
+      "logits/rejected": 77.16484069824219,
+      "logps/chosen": -28.32808494567871,
+      "logps/rejected": -32.60577392578125,
+      "loss": 0.4534,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.20245198905467987,
+      "rewards/margins": 0.19954310357570648,
+      "rewards/rejected": 0.002908907597884536,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 10.625,
+      "grad_norm": 3.046875,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 81.62095642089844,
-      "logits/rejected": 81.64241027832031,
-      "logps/chosen": -31.521636962890625,
-      "logps/rejected": -33.3758659362793,
-      "loss": 0.377,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.6762043833732605,
-      "rewards/margins": 0.6790729761123657,
-      "rewards/rejected": -0.002868417650461197,
+      "logits/chosen": 81.41078186035156,
+      "logits/rejected": 81.44329071044922,
+      "logps/chosen": -31.582672119140625,
+      "logps/rejected": -33.35100555419922,
+      "loss": 0.4655,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16294743120670319,
+      "rewards/margins": 0.16117897629737854,
+      "rewards/rejected": 0.0017684675985947251,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 7.71875,
+      "grad_norm": 2.34375,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 80.68013763427734,
-      "logits/rejected": 80.69649505615234,
-      "logps/chosen": -31.641170501708984,
-      "logps/rejected": -32.8623161315918,
-      "loss": 0.3712,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.8010267019271851,
-      "rewards/margins": 0.7092341780662537,
-      "rewards/rejected": 0.0917925089597702,
+      "logits/chosen": 80.47315216064453,
+      "logits/rejected": 80.48896789550781,
+      "logps/chosen": -31.6666316986084,
+      "logps/rejected": -32.75847625732422,
+      "loss": 0.4641,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1977105438709259,
+      "rewards/margins": 0.1643778532743454,
+      "rewards/rejected": 0.03333265334367752,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 6.25,
+      "grad_norm": 2.5625,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 82.13801574707031,
-      "logits/rejected": 82.19556427001953,
-      "logps/chosen": -27.77614402770996,
-      "logps/rejected": -31.526113510131836,
-      "loss": 0.369,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.7104269862174988,
-      "rewards/margins": 0.6609331965446472,
-      "rewards/rejected": 0.04949387162923813,
+      "logits/chosen": 81.98124694824219,
+      "logits/rejected": 82.04177856445312,
+      "logps/chosen": -27.832752227783203,
+      "logps/rejected": -31.519290924072266,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.17194589972496033,
+      "rewards/margins": 0.15889012813568115,
+      "rewards/rejected": 0.013055793941020966,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 7.5,
+      "grad_norm": 2.609375,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 81.63580322265625,
-      "logits/rejected": 81.66941833496094,
-      "logps/chosen": -31.2968692779541,
-      "logps/rejected": -34.87516403198242,
-      "loss": 0.4021,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.6317180395126343,
-      "rewards/margins": 0.5380347967147827,
-      "rewards/rejected": 0.09368324279785156,
+      "logits/chosen": 81.46924591064453,
+      "logits/rejected": 81.51275634765625,
+      "logps/chosen": -31.354238510131836,
+      "logps/rejected": -34.80229568481445,
+      "loss": 0.4735,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.15219244360923767,
+      "rewards/margins": 0.1214846819639206,
+      "rewards/rejected": 0.03070778027176857,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 8.625,
+      "grad_norm": 2.546875,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 75.4756088256836,
-      "logits/rejected": 75.35409545898438,
-      "logps/chosen": -29.19217300415039,
-      "logps/rejected": -27.851299285888672,
-      "loss": 0.4126,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.5535811185836792,
-      "rewards/margins": 0.4298717975616455,
-      "rewards/rejected": 0.1237092986702919,
+      "logits/chosen": 75.28022766113281,
+      "logits/rejected": 75.15403747558594,
+      "logps/chosen": -28.978900909423828,
+      "logps/rejected": -27.919509887695312,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.15972287952899933,
+      "rewards/margins": 0.13561637699604034,
+      "rewards/rejected": 0.024106483906507492,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.4199758805237807,
-      "train_runtime": 2559.5376,
-      "train_samples_per_second": 1.203,
-      "train_steps_per_second": 0.15
+      "train_loss": 0.47652822160101554,
+      "train_runtime": 2558.0485,
+      "train_samples_per_second": 1.204,
+      "train_steps_per_second": 0.151
     }
   ],
   "logging_steps": 10,