Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9944
-- Rewards/chosen: -0.0085
-- Rewards/rejected: -0.0141
-- Rewards/accuracies: 0.6005
-- Rewards/margins: 0.0056
-- Logps/rejected: -37.3726
-- Logps/chosen: -33.2900
-- Logits/rejected: 97.6266
-- Logits/chosen: 97.6599
+- Loss: 1.1081
+- Rewards/chosen: -0.0561
+- Rewards/rejected: -0.0772
+- Rewards/accuracies: 0.4776
+- Rewards/margins: 0.0212
+- Logps/rejected: -36.0524
+- Logps/chosen: -32.5055
+- Logits/rejected: 98.7108
+- Logits/chosen: 98.7301
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9955        | 0.26  | 100  | 0.9999          | 0.0005         | 0.0004           | 0.5187             | 0.0001          | -35.9246       | -32.3899     | 98.7274         | 98.7339       |
-| 0.9858        | 0.52  | 200  | 0.9965          | -0.0053        | -0.0088          | 0.5274             | 0.0035          | -36.8453       | -32.9718     | 98.0264         | 98.0513       |
-| 0.9814        | 0.78  | 300  | 0.9947          | -0.0086        | -0.0139          | 0.5889             | 0.0053          | -37.3559       | -33.3009     | 97.6402         | 97.6739       |
+| 0.9852        | 0.26  | 100  | 1.1083          | -0.0231        | -0.0451          | 0.4875             | 0.0220          | -36.0166       | -32.4688     | 98.6830         | 98.6942       |
+| 0.902         | 0.52  | 200  | 0.9846          | -0.1464        | -0.3760          | 0.5548             | 0.2296          | -36.3844       | -32.6059     | 98.6271         | 98.6562       |
+| 0.671         | 0.78  | 300  | 1.1081          | -0.0561        | -0.0772          | 0.4776             | 0.0212          | -36.0524       | -32.5055     | 98.7108         | 98.7301       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63d7809bbd6f6b4d5e6d84492d93491d0ebc7d7763bb1918705421d2bb83f327
+oid sha256:533058f632960fe234a13bda36d0dd3f02dc7ee2c5a8441e28396f677c2a971e
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 97.65992736816406,
-    "eval_logits/rejected": 97.62660217285156,
-    "eval_logps/chosen": -33.290000915527344,
-    "eval_logps/rejected": -37.37261199951172,
-    "eval_loss": 0.994398832321167,
-    "eval_rewards/accuracies": 0.6004983186721802,
-    "eval_rewards/chosen": -0.008468217216432095,
-    "eval_rewards/margins": 0.005592645611613989,
-    "eval_rewards/rejected": -0.014060864225029945,
-    "eval_runtime": 103.8235,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.304,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.9878765378679548,
-    "train_runtime": 2559.7639,
+    "train_loss": 0.7878170496457583,
+    "train_runtime": 2553.4781,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.206,
+    "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9878765378679548,
-    "train_runtime": 2559.7639,
+    "train_loss": 0.7878170496457583,
+    "train_runtime": 2553.4781,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.206,
+    "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1.3046875,
+      "grad_norm": 118.0,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 1.09375,
+      "grad_norm": 97.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.09645080566406,
-      "logits/rejected": 80.80389404296875,
-      "logps/chosen": -34.27156066894531,
-      "logps/rejected": -33.039093017578125,
-      "loss": 0.9995,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": -0.00029834467568434775,
-      "rewards/margins": 0.0005084889708086848,
-      "rewards/rejected": -0.0008068337920121849,
+      "logits/chosen": 81.08590698242188,
+      "logits/rejected": 80.79015350341797,
+      "logps/chosen": -34.172550201416016,
+      "logps/rejected": -32.99056625366211,
+      "loss": 0.9305,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.062255993485450745,
+      "rewards/margins": 0.09120028465986252,
+      "rewards/rejected": -0.028944293037056923,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 1.2734375,
+      "grad_norm": 83.5,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.66552734375,
-      "logits/rejected": 80.5560073852539,
-      "logps/chosen": -33.4774055480957,
-      "logps/rejected": -30.691213607788086,
-      "loss": 0.9987,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0021250424906611443,
-      "rewards/margins": 0.0012622694484889507,
-      "rewards/rejected": 0.000862772751133889,
+      "logits/chosen": 80.69185638427734,
+      "logits/rejected": 80.58460235595703,
+      "logps/chosen": -33.645851135253906,
+      "logps/rejected": -30.73211097717285,
+      "loss": 1.0725,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.039649851620197296,
+      "rewards/margins": -0.0011898368829861283,
+      "rewards/rejected": 0.04083969444036484,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 1.2109375,
+      "grad_norm": 105.5,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.51115417480469,
-      "logits/rejected": 82.54508972167969,
-      "logps/chosen": -33.80036926269531,
-      "logps/rejected": -31.189748764038086,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.002124499063938856,
-      "rewards/margins": -0.00021631647541653365,
-      "rewards/rejected": 0.0023408152628690004,
+      "logits/chosen": 82.52629089355469,
+      "logits/rejected": 82.55845642089844,
+      "logps/chosen": -33.839778900146484,
+      "logps/rejected": -31.187374114990234,
+      "loss": 1.1222,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.15573647618293762,
+      "rewards/margins": -0.05707378312945366,
+      "rewards/rejected": 0.21281024813652039,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 1.1796875,
+      "grad_norm": 83.5,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.10090637207031,
-      "logits/rejected": 81.09576416015625,
-      "logps/chosen": -32.7674560546875,
-      "logps/rejected": -33.11550521850586,
-      "loss": 0.9985,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0040660640224814415,
-      "rewards/margins": 0.0015358638484030962,
-      "rewards/rejected": 0.0025301999412477016,
+      "logits/chosen": 81.00019073486328,
+      "logits/rejected": 80.99311828613281,
+      "logps/chosen": -32.78485870361328,
+      "logps/rejected": -33.173404693603516,
+      "loss": 1.023,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.35028138756752014,
+      "rewards/margins": 0.17467467486858368,
+      "rewards/rejected": 0.17560675740242004,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 1.1875,
+      "grad_norm": 72.5,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.85154724121094,
-      "logits/rejected": 78.85734558105469,
-      "logps/chosen": -30.360393524169922,
-      "logps/rejected": -30.609283447265625,
-      "loss": 0.9962,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.008448543958365917,
-      "rewards/margins": 0.0038085163105279207,
-      "rewards/rejected": 0.0046400283463299274,
+      "logits/chosen": 78.55744934082031,
+      "logits/rejected": 78.57440948486328,
+      "logps/chosen": -30.58083724975586,
+      "logps/rejected": -30.754711151123047,
+      "loss": 1.0219,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.5619677901268005,
+      "rewards/margins": 0.2752513289451599,
+      "rewards/rejected": 0.2867165207862854,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 0.96484375,
+      "grad_norm": 94.5,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.49021911621094,
-      "logits/rejected": 83.54866027832031,
-      "logps/chosen": -30.763973236083984,
-      "logps/rejected": -29.17538833618164,
-      "loss": 1.0,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 0.004111888352781534,
-      "rewards/margins": -3.9446913433494046e-05,
-      "rewards/rejected": 0.004151335451751947,
+      "logits/chosen": 83.01163482666016,
+      "logits/rejected": 83.06632995605469,
+      "logps/chosen": -30.960674285888672,
+      "logps/rejected": -29.359949111938477,
+      "loss": 1.1647,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.19303548336029053,
+      "rewards/margins": -0.014480452053248882,
+      "rewards/rejected": 0.20751595497131348,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 1.2109375,
+      "grad_norm": 116.5,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 84.11228942871094,
-      "logits/rejected": 84.1441650390625,
-      "logps/chosen": -30.222454071044922,
-      "logps/rejected": -32.666595458984375,
-      "loss": 0.9996,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0040657008066773415,
-      "rewards/margins": 0.0003545849467627704,
-      "rewards/rejected": 0.003711115103214979,
+      "logits/chosen": 83.62989044189453,
+      "logits/rejected": 83.66007232666016,
+      "logps/chosen": -30.669225692749023,
+      "logps/rejected": -33.12618637084961,
+      "loss": 1.1523,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.03618011251091957,
+      "rewards/margins": 0.04344985634088516,
+      "rewards/rejected": -0.07962997257709503,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 1.2421875,
+      "grad_norm": 85.0,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.86946868896484,
-      "logits/rejected": 81.84814453125,
-      "logps/chosen": -30.959096908569336,
-      "logps/rejected": -30.652545928955078,
-      "loss": 0.9956,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.005458436906337738,
-      "rewards/margins": 0.0044073979370296,
-      "rewards/rejected": 0.001051038852892816,
+      "logits/chosen": 81.133056640625,
+      "logits/rejected": 81.11943054199219,
+      "logps/chosen": -31.318958282470703,
+      "logps/rejected": -30.970218658447266,
+      "loss": 0.845,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.167384073138237,
+      "rewards/margins": 0.3586970567703247,
+      "rewards/rejected": -0.1913129985332489,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 1.484375,
+      "grad_norm": 109.0,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.68418884277344,
-      "logits/rejected": 78.65721893310547,
-      "logps/chosen": -32.17829513549805,
-      "logps/rejected": -30.884775161743164,
-      "loss": 0.9961,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.00459885410964489,
-      "rewards/margins": 0.00393189862370491,
-      "rewards/rejected": 0.0006669552531093359,
+      "logits/chosen": 77.96097564697266,
+      "logits/rejected": 77.93636322021484,
+      "logps/chosen": -32.162574768066406,
+      "logps/rejected": -31.054067611694336,
+      "loss": 0.7906,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.4280422329902649,
+      "rewards/margins": 0.5203782916069031,
+      "rewards/rejected": -0.09233605861663818,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 1.2421875,
+      "grad_norm": 68.5,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.70716857910156,
-      "logits/rejected": 83.73751068115234,
-      "logps/chosen": -33.73701477050781,
-      "logps/rejected": -31.63702964782715,
-      "loss": 0.9955,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.005679761990904808,
-      "rewards/margins": 0.004544637631624937,
-      "rewards/rejected": 0.0011351245921105146,
+      "logits/chosen": 83.30989837646484,
+      "logits/rejected": 83.33601379394531,
+      "logps/chosen": -33.85996627807617,
+      "logps/rejected": -31.64011573791504,
+      "loss": 0.9852,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.4005257189273834,
+      "rewards/margins": 0.30113959312438965,
+      "rewards/rejected": 0.09938610345125198,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.73394012451172,
-      "eval_logits/rejected": 98.7273941040039,
-      "eval_logps/chosen": -32.38990783691406,
-      "eval_logps/rejected": -35.92463684082031,
-      "eval_loss": 0.999876856803894,
-      "eval_rewards/accuracies": 0.5186877250671387,
-      "eval_rewards/chosen": 0.0005326389218680561,
-      "eval_rewards/margins": 0.00011375291069271043,
-      "eval_rewards/rejected": 0.00041888616397045553,
-      "eval_runtime": 104.2424,
-      "eval_samples_per_second": 3.29,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.69417572021484,
+      "eval_logits/rejected": 98.68303680419922,
+      "eval_logps/chosen": -32.46882629394531,
+      "eval_logps/rejected": -36.016597747802734,
+      "eval_loss": 1.1082578897476196,
+      "eval_rewards/accuracies": 0.4875415563583374,
+      "eval_rewards/chosen": -0.02308560535311699,
+      "eval_rewards/margins": 0.021978026255965233,
+      "eval_rewards/rejected": -0.04506362974643707,
+      "eval_runtime": 104.3924,
+      "eval_samples_per_second": 3.286,
+      "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 1.4140625,
+      "grad_norm": 100.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.85816955566406,
-      "logits/rejected": 83.75128936767578,
-      "logps/chosen": -32.19211959838867,
-      "logps/rejected": -32.65901565551758,
-      "loss": 0.9917,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.008306830190122128,
-      "rewards/margins": 0.008338114246726036,
-      "rewards/rejected": -3.12842421408277e-05,
+      "logits/chosen": 83.43294525146484,
+      "logits/rejected": 83.3353042602539,
+      "logps/chosen": -32.524871826171875,
+      "logps/rejected": -32.74653625488281,
+      "loss": 0.8192,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.4481372833251953,
+      "rewards/margins": 0.5297205448150635,
+      "rewards/rejected": -0.08158326148986816,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 1.3046875,
+      "grad_norm": 86.5,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.82106018066406,
-      "logits/rejected": 83.92265319824219,
-      "logps/chosen": -28.150625228881836,
-      "logps/rejected": -35.3939208984375,
-      "loss": 0.9929,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.007789556868374348,
-      "rewards/margins": 0.0071373311802744865,
-      "rewards/rejected": 0.000652224407531321,
+      "logits/chosen": 83.61913299560547,
+      "logits/rejected": 83.73504638671875,
+      "logps/chosen": -28.402780532836914,
+      "logps/rejected": -35.50286865234375,
+      "loss": 0.7691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.4741190969944,
+      "rewards/margins": 0.5134681463241577,
+      "rewards/rejected": -0.039348993450403214,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 0.9453125,
+      "grad_norm": 72.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 80.96563720703125,
-      "logits/rejected": 80.99563598632812,
-      "logps/chosen": -30.216140747070312,
-      "logps/rejected": -31.844036102294922,
-      "loss": 0.994,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.006912143435329199,
-      "rewards/margins": 0.006036223843693733,
-      "rewards/rejected": 0.0008759202319197357,
+      "logits/chosen": 80.9137954711914,
+      "logits/rejected": 80.93389892578125,
+      "logps/chosen": -30.521535873413086,
+      "logps/rejected": -32.07093048095703,
+      "loss": 0.815,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.34723615646362305,
+      "rewards/margins": 0.47260579466819763,
+      "rewards/rejected": -0.12536963820457458,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 1.1171875,
+      "grad_norm": 86.0,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 81.863525390625,
-      "logits/rejected": 81.86921691894531,
-      "logps/chosen": -26.845142364501953,
-      "logps/rejected": -33.07027816772461,
-      "loss": 0.9875,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.006021805107593536,
-      "rewards/margins": 0.012490840628743172,
-      "rewards/rejected": -0.006469034589827061,
+      "logits/chosen": 82.1353530883789,
+      "logits/rejected": 82.16053771972656,
+      "logps/chosen": -27.094701766967773,
+      "logps/rejected": -32.902889251708984,
+      "loss": 0.7235,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.3173602223396301,
+      "rewards/margins": 0.7489217519760132,
+      "rewards/rejected": -0.4315616488456726,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 1.21875,
+      "grad_norm": 59.75,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.13746643066406,
-      "logits/rejected": 80.10902404785156,
-      "logps/chosen": -28.976547241210938,
-      "logps/rejected": -33.208518981933594,
-      "loss": 0.9895,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.004125660751014948,
-      "rewards/margins": 0.010545835830271244,
-      "rewards/rejected": -0.006420175079256296,
+      "logits/chosen": 80.8051986694336,
+      "logits/rejected": 80.77262115478516,
+      "logps/chosen": -28.932445526123047,
+      "logps/rejected": -33.024757385253906,
+      "loss": 0.633,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.4109979569911957,
+      "rewards/margins": 0.8234249949455261,
+      "rewards/rejected": -0.4124270975589752,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 1.3515625,
+      "grad_norm": 82.5,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 81.72142028808594,
-      "logits/rejected": 81.74298858642578,
-      "logps/chosen": -33.8978157043457,
-      "logps/rejected": -30.907711029052734,
-      "loss": 0.9881,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.002389371395111084,
-      "rewards/margins": 0.011941083706915379,
-      "rewards/rejected": -0.009551710449159145,
+      "logits/chosen": 82.64894104003906,
+      "logits/rejected": 82.64643859863281,
+      "logps/chosen": -33.55781936645508,
+      "logps/rejected": -30.31070899963379,
+      "loss": 0.7859,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5210397839546204,
+      "rewards/margins": 0.8433935046195984,
+      "rewards/rejected": -0.3223537802696228,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 1.4765625,
+      "grad_norm": 81.5,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 82.35487365722656,
-      "logits/rejected": 82.30474090576172,
-      "logps/chosen": -30.870525360107422,
-      "logps/rejected": -33.04078674316406,
-      "loss": 0.9859,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.003887615632265806,
-      "rewards/margins": 0.014094889163970947,
-      "rewards/rejected": -0.010207273997366428,
+      "logits/chosen": 83.43087005615234,
+      "logits/rejected": 83.37384033203125,
+      "logps/chosen": -30.71698570251465,
+      "logps/rejected": -32.415626525878906,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4880717396736145,
+      "rewards/margins": 0.8440794944763184,
+      "rewards/rejected": -0.35600775480270386,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 1.21875,
+      "grad_norm": 64.0,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 79.4852066040039,
-      "logits/rejected": 79.46187591552734,
-      "logps/chosen": -31.02083396911621,
-      "logps/rejected": -32.165191650390625,
-      "loss": 0.9882,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.0017631975933909416,
-      "rewards/margins": 0.011843027547001839,
-      "rewards/rejected": -0.010079829022288322,
+      "logits/chosen": 81.02064514160156,
+      "logits/rejected": 81.00648498535156,
+      "logps/chosen": -30.46847152709961,
+      "logps/rejected": -31.655284881591797,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.6558108925819397,
+      "rewards/margins": 1.104081153869629,
+      "rewards/rejected": -0.4482702314853668,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 0.9609375,
+      "grad_norm": 27.25,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 80.97419738769531,
-      "logits/rejected": 80.94820404052734,
-      "logps/chosen": -30.60634994506836,
-      "logps/rejected": -31.083566665649414,
-      "loss": 0.9922,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0016006485093384981,
-      "rewards/margins": 0.007756076753139496,
-      "rewards/rejected": -0.006155428942292929,
+      "logits/chosen": 82.70450592041016,
+      "logits/rejected": 82.70726776123047,
+      "logps/chosen": -30.44081687927246,
+      "logps/rejected": -30.764562606811523,
+      "loss": 0.8467,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.2930375039577484,
+      "rewards/margins": 0.5599225759506226,
+      "rewards/rejected": -0.2668851315975189,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 1.453125,
+      "grad_norm": 68.0,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 76.12115478515625,
-      "logits/rejected": 76.07009887695312,
-      "logps/chosen": -34.18424606323242,
-      "logps/rejected": -33.341392517089844,
-      "loss": 0.9858,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.005684881471097469,
-      "rewards/margins": 0.014164777472615242,
-      "rewards/rejected": -0.008479896001517773,
+      "logits/chosen": 78.14289093017578,
+      "logits/rejected": 78.0774917602539,
+      "logps/chosen": -34.14347839355469,
+      "logps/rejected": -32.717750549316406,
+      "loss": 0.902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.5483377575874329,
+      "rewards/margins": 0.7502498030662537,
+      "rewards/rejected": -0.2019120752811432,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.05126190185547,
-      "eval_logits/rejected": 98.02639770507812,
-      "eval_logps/chosen": -32.97175979614258,
-      "eval_logps/rejected": -36.845333099365234,
-      "eval_loss": 0.9964954853057861,
-      "eval_rewards/accuracies": 0.5274086594581604,
-      "eval_rewards/chosen": -0.005285844672471285,
-      "eval_rewards/margins": 0.0035022026859223843,
-      "eval_rewards/rejected": -0.008788047358393669,
-      "eval_runtime": 104.1082,
-      "eval_samples_per_second": 3.295,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.65617370605469,
+      "eval_logits/rejected": 98.62713623046875,
+      "eval_logps/chosen": -32.60585021972656,
+      "eval_logps/rejected": -36.384361267089844,
+      "eval_loss": 0.9846288561820984,
+      "eval_rewards/accuracies": 0.554817259311676,
+      "eval_rewards/chosen": -0.14640627801418304,
+      "eval_rewards/margins": 0.22964100539684296,
+      "eval_rewards/rejected": -0.3760472238063812,
+      "eval_runtime": 103.9231,
+      "eval_samples_per_second": 3.301,
+      "eval_steps_per_second": 0.414,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 1.84375,
+      "grad_norm": 107.0,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 78.64119720458984,
-      "logits/rejected": 78.55430603027344,
-      "logps/chosen": -33.689414978027344,
-      "logps/rejected": -36.20193862915039,
-      "loss": 0.9853,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.002563029993325472,
-      "rewards/margins": 0.014721485786139965,
-      "rewards/rejected": -0.012158457189798355,
+      "logits/chosen": 80.76277160644531,
+      "logits/rejected": 80.67192840576172,
+      "logps/chosen": -33.424560546875,
+      "logps/rejected": -35.27748107910156,
+      "loss": 0.7859,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4690447747707367,
+      "rewards/margins": 0.7312911152839661,
+      "rewards/rejected": -0.26224634051322937,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 1.484375,
+      "grad_norm": 86.0,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 80.5840835571289,
-      "logits/rejected": 80.67861938476562,
-      "logps/chosen": -31.57720947265625,
-      "logps/rejected": -31.91719627380371,
-      "loss": 0.9844,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.003717451822012663,
-      "rewards/margins": 0.015599893406033516,
-      "rewards/rejected": -0.011882440187036991,
+      "logits/chosen": 82.9009017944336,
+      "logits/rejected": 82.97982788085938,
+      "logps/chosen": -31.129741668701172,
+      "logps/rejected": -31.022235870361328,
+      "loss": 0.566,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7372905611991882,
+      "rewards/margins": 1.001245141029358,
+      "rewards/rejected": -0.2639545798301697,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 1.3984375,
+      "grad_norm": 79.0,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 77.53582763671875,
-      "logits/rejected": 77.5838851928711,
-      "logps/chosen": -32.72165298461914,
-      "logps/rejected": -35.34224319458008,
-      "loss": 0.9859,
+      "logits/chosen": 80.10737609863281,
+      "logits/rejected": 80.16769409179688,
+      "logps/chosen": -32.2999153137207,
+      "logps/rejected": -34.19389343261719,
+      "loss": 0.904,
       "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0008409392321482301,
-      "rewards/margins": 0.014141863211989403,
-      "rewards/rejected": -0.013300922699272633,
+      "rewards/chosen": 0.45524734258651733,
+      "rewards/margins": 0.6188174486160278,
+      "rewards/rejected": -0.16357013583183289,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 1.671875,
+      "grad_norm": 101.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 79.64659118652344,
-      "logits/rejected": 79.958984375,
-      "logps/chosen": -31.332469940185547,
-      "logps/rejected": -32.86049270629883,
-      "loss": 0.9832,
+      "logits/chosen": 82.58370208740234,
+      "logits/rejected": 82.85044860839844,
+      "logps/chosen": -30.799701690673828,
+      "logps/rejected": -31.604589462280273,
+      "loss": 0.7596,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.003555959090590477,
-      "rewards/margins": 0.016840480268001556,
-      "rewards/rejected": -0.01328451931476593,
+      "rewards/chosen": 0.7995287179946899,
+      "rewards/margins": 0.8648217916488647,
+      "rewards/rejected": -0.06529306620359421,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 1.484375,
+      "grad_norm": 89.5,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 78.12522888183594,
-      "logits/rejected": 78.17500305175781,
-      "logps/chosen": -27.822484970092773,
-      "logps/rejected": -31.34881019592285,
-      "loss": 0.9878,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.002265883143991232,
-      "rewards/margins": 0.012214846909046173,
-      "rewards/rejected": -0.014480730518698692,
+      "logits/chosen": 81.37136840820312,
+      "logits/rejected": 81.4355697631836,
+      "logps/chosen": -26.921367645263672,
+      "logps/rejected": -30.0640811920166,
+      "loss": 0.8158,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.6070747971534729,
+      "rewards/margins": 0.754082202911377,
+      "rewards/rejected": -0.14700737595558167,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 1.6171875,
+      "grad_norm": 69.0,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 75.08821105957031,
-      "logits/rejected": 75.22389221191406,
-      "logps/chosen": -31.112863540649414,
-      "logps/rejected": -38.40215301513672,
-      "loss": 0.9725,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.004481295123696327,
-      "rewards/margins": 0.027486557140946388,
-      "rewards/rejected": -0.023005260154604912,
+      "logits/chosen": 78.6139144897461,
+      "logits/rejected": 78.74816131591797,
+      "logps/chosen": -30.19488525390625,
+      "logps/rejected": -36.41002655029297,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.2295000553131104,
+      "rewards/margins": 1.5070557594299316,
+      "rewards/rejected": -0.2775557041168213,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 1.3515625,
+      "grad_norm": 36.0,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 74.10564422607422,
-      "logits/rejected": 74.13673400878906,
-      "logps/chosen": -32.07135772705078,
-      "logps/rejected": -33.231197357177734,
-      "loss": 0.9857,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.002924318192526698,
-      "rewards/margins": 0.01433342695236206,
-      "rewards/rejected": -0.017257746309041977,
+      "logits/chosen": 78.08236694335938,
+      "logits/rejected": 78.10992431640625,
+      "logps/chosen": -30.87982749938965,
+      "logps/rejected": -31.692874908447266,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.8091908693313599,
+      "rewards/margins": 0.977896511554718,
+      "rewards/rejected": -0.16870568692684174,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 2.03125,
+      "grad_norm": 113.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 77.0625991821289,
-      "logits/rejected": 76.84493255615234,
-      "logps/chosen": -32.438629150390625,
-      "logps/rejected": -31.16558265686035,
-      "loss": 0.9913,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.006287367548793554,
-      "rewards/margins": 0.008698503486812115,
-      "rewards/rejected": -0.014985869638621807,
+      "logits/chosen": 80.7326431274414,
+      "logits/rejected": 80.51844024658203,
+      "logps/chosen": -31.088062286376953,
+      "logps/rejected": -29.66598892211914,
+      "loss": 0.9291,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.6496511697769165,
+      "rewards/margins": 0.6487425565719604,
+      "rewards/rejected": 0.0009086370700970292,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 1.34375,
+      "grad_norm": 72.0,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 77.05482482910156,
-      "logits/rejected": 76.97974395751953,
-      "logps/chosen": -34.25292205810547,
-      "logps/rejected": -34.649898529052734,
-      "loss": 0.9752,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.0014867703430354595,
-      "rewards/margins": 0.02484356239438057,
-      "rewards/rejected": -0.026330333203077316,
+      "logits/chosen": 80.9128189086914,
+      "logits/rejected": 80.82886505126953,
+      "logps/chosen": -33.01869201660156,
+      "logps/rejected": -32.421485900878906,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.977002739906311,
+      "rewards/margins": 1.3411604166030884,
+      "rewards/rejected": -0.3641577661037445,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 1.4921875,
+      "grad_norm": 70.5,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 72.6989974975586,
-      "logits/rejected": 72.833984375,
-      "logps/chosen": -33.264137268066406,
-      "logps/rejected": -30.818592071533203,
-      "loss": 0.9814,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.0014156814431771636,
-      "rewards/margins": 0.018581366166472435,
-      "rewards/rejected": -0.017165686935186386,
+      "logits/chosen": 76.50440979003906,
+      "logits/rejected": 76.59843444824219,
+      "logps/chosen": -32.13983917236328,
+      "logps/rejected": -29.128122329711914,
+      "loss": 0.671,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.139281988143921,
+      "rewards/margins": 1.1627700328826904,
+      "rewards/rejected": -0.023488014936447144,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 97.67390441894531,
-      "eval_logits/rejected": 97.64021301269531,
-      "eval_logps/chosen": -33.30087661743164,
-      "eval_logps/rejected": -37.35591125488281,
-      "eval_loss": 0.99467533826828,
-      "eval_rewards/accuracies": 0.5888704061508179,
-      "eval_rewards/chosen": -0.008576988242566586,
-      "eval_rewards/margins": 0.00531682837754488,
-      "eval_rewards/rejected": -0.01389381755143404,
-      "eval_runtime": 103.9426,
-      "eval_samples_per_second": 3.3,
+      "eval_logits/chosen": 98.7301254272461,
+      "eval_logits/rejected": 98.71080017089844,
+      "eval_logps/chosen": -32.50548553466797,
+      "eval_logps/rejected": -36.0523567199707,
+      "eval_loss": 1.1081271171569824,
+      "eval_rewards/accuracies": 0.47757473587989807,
+      "eval_rewards/chosen": -0.056078068912029266,
+      "eval_rewards/margins": 0.021167948842048645,
+      "eval_rewards/rejected": -0.07724600285291672,
+      "eval_runtime": 103.8638,
+      "eval_samples_per_second": 3.302,
       "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 1.609375,
+      "grad_norm": 66.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 80.20366668701172,
-      "logits/rejected": 80.20387268066406,
-      "logps/chosen": -30.933481216430664,
-      "logps/rejected": -34.256614685058594,
-      "loss": 0.9801,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0006994610885158181,
-      "rewards/margins": 0.019869104027748108,
-      "rewards/rejected": -0.020568564534187317,
+      "logits/chosen": 83.55355072021484,
+      "logits/rejected": 83.5801773071289,
+      "logps/chosen": -30.023509979248047,
+      "logps/rejected": -32.43012237548828,
+      "loss": 0.6531,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.7560194730758667,
+      "rewards/margins": 0.9633440971374512,
+      "rewards/rejected": -0.20732466876506805,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 1.5390625,
+      "grad_norm": 56.25,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 77.26924133300781,
-      "logits/rejected": 77.28764343261719,
-      "logps/chosen": -31.467296600341797,
-      "logps/rejected": -30.592571258544922,
-      "loss": 0.982,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0013126448029652238,
-      "rewards/margins": 0.01804344728589058,
-      "rewards/rejected": -0.01673080213367939,
+      "logits/chosen": 81.07228088378906,
+      "logits/rejected": 81.07295989990234,
+      "logps/chosen": -30.569133758544922,
+      "logps/rejected": -29.126922607421875,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.926487147808075,
+      "rewards/margins": 1.1131722927093506,
+      "rewards/rejected": -0.1866852045059204,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 1.875,
+      "grad_norm": 65.5,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 74.20513916015625,
-      "logits/rejected": 74.25221252441406,
-      "logps/chosen": -29.909320831298828,
-      "logps/rejected": -34.725521087646484,
-      "loss": 0.9747,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.004432853776961565,
-      "rewards/margins": 0.025339430198073387,
-      "rewards/rejected": -0.02090657688677311,
+      "logits/chosen": 78.3108139038086,
+      "logits/rejected": 78.35076904296875,
+      "logps/chosen": -29.139026641845703,
+      "logps/rejected": -32.85981750488281,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.0922209024429321,
+      "rewards/margins": 1.294679045677185,
+      "rewards/rejected": -0.20245835185050964,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 1.7578125,
+      "grad_norm": 98.5,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 78.9554672241211,
-      "logits/rejected": 78.990478515625,
-      "logps/chosen": -33.333351135253906,
-      "logps/rejected": -35.866722106933594,
-      "loss": 0.9762,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.0012120162136852741,
-      "rewards/margins": 0.023768287152051926,
-      "rewards/rejected": -0.024980302900075912,
+      "logits/chosen": 82.51289367675781,
+      "logits/rejected": 82.52921295166016,
+      "logps/chosen": -32.200294494628906,
+      "logps/rejected": -33.6099853515625,
+      "loss": 0.7511,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.910667896270752,
+      "rewards/margins": 1.127828598022461,
+      "rewards/rejected": -0.21716061234474182,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 1.5859375,
+      "grad_norm": 47.0,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 77.8236083984375,
-      "logits/rejected": 77.84061431884766,
-      "logps/chosen": -33.3131103515625,
-      "logps/rejected": -35.05299758911133,
-      "loss": 0.9771,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.0033062633592635393,
-      "rewards/margins": 0.0229182131588459,
-      "rewards/rejected": -0.01961195096373558,
+      "logits/chosen": 81.58796691894531,
+      "logits/rejected": 81.5913314819336,
+      "logps/chosen": -32.477813720703125,
+      "logps/rejected": -33.15494155883789,
+      "loss": 0.6388,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.049329400062561,
+      "rewards/margins": 1.106154203414917,
+      "rewards/rejected": -0.056824732571840286,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 1.5234375,
+      "grad_norm": 59.0,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 79.53218078613281,
-      "logits/rejected": 79.56050109863281,
-      "logps/chosen": -29.001379013061523,
-      "logps/rejected": -33.16984176635742,
-      "loss": 0.9793,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.005508318077772856,
-      "rewards/margins": 0.020708225667476654,
-      "rewards/rejected": -0.015199905261397362,
+      "logits/chosen": 83.0643539428711,
+      "logits/rejected": 83.09364318847656,
+      "logps/chosen": -28.514789581298828,
+      "logps/rejected": -31.751323699951172,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.9336811304092407,
+      "rewards/margins": 1.0250084400177002,
+      "rewards/rejected": -0.09132737666368484,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 1.7890625,
+      "grad_norm": 121.5,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 78.969970703125,
-      "logits/rejected": 78.9748306274414,
-      "logps/chosen": -33.2999267578125,
-      "logps/rejected": -37.32087326049805,
-      "loss": 0.9821,
+      "logits/chosen": 82.48683166503906,
+      "logits/rejected": 82.50460815429688,
+      "logps/chosen": -31.874202728271484,
+      "logps/rejected": -35.20905303955078,
+      "loss": 0.7031,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.004237635992467403,
-      "rewards/margins": 0.017877381294965744,
-      "rewards/rejected": -0.022115018218755722,
+      "rewards/chosen": 0.9017645120620728,
+      "rewards/margins": 0.9914749264717102,
+      "rewards/rejected": -0.08971036225557327,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 1.359375,
+      "grad_norm": 94.5,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 72.49492645263672,
-      "logits/rejected": 72.36249542236328,
-      "logps/chosen": -30.828378677368164,
-      "logps/rejected": -29.885875701904297,
-      "loss": 0.9853,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0025225188583135605,
-      "rewards/margins": 0.014730495400726795,
-      "rewards/rejected": -0.01725301705300808,
+      "logits/chosen": 76.47808074951172,
+      "logits/rejected": 76.34886932373047,
+      "logps/chosen": -29.865795135498047,
+      "logps/rejected": -28.233165740966797,
+      "loss": 0.7695,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.6392983794212341,
+      "rewards/margins": 0.7046308517456055,
+      "rewards/rejected": -0.06533239781856537,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9878765378679548,
-      "train_runtime": 2559.7639,
-      "train_samples_per_second": 1.203,
-      "train_steps_per_second": 0.15
+      "train_loss": 0.7878170496457583,
+      "train_runtime": 2553.4781,
+      "train_samples_per_second": 1.206,
+      "train_steps_per_second": 0.151
     }
   ],
   "logging_steps": 10,