Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1077
-- Rewards/chosen: -0.0230
-- Rewards/rejected: -0.0718
-- Rewards/accuracies: 0.4988
-- Rewards/margins: 0.0488
-- Logps/rejected: -36.0463
-- Logps/chosen: -32.4687
-- Logits/rejected: 98.7000
-- Logits/chosen: 98.7211
+- Loss: 0.7391
+- Rewards/chosen: -0.0991
+- Rewards/rejected: -0.2198
+- Rewards/accuracies: 0.5486
+- Rewards/margins: 0.1207
+- Logps/rejected: -36.2805
+- Logps/chosen: -32.5848
+- Logits/rejected: 98.6399
+- Logits/chosen: 98.6637
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9852        | 0.26  | 100  | 1.1083          | -0.0231        | -0.0451          | 0.4875             | 0.0220          | -36.0166       | -32.4688     | 98.6830         | 98.6942       |
-| 0.902         | 0.52  | 200  | 0.9846          | -0.1464        | -0.3760          | 0.5548             | 0.2296          | -36.3844       | -32.6059     | 98.6271         | 98.6562       |
-| 0.671         | 0.78  | 300  | 1.1081          | -0.0561        | -0.0772          | 0.4776             | 0.0212          | -36.0524       | -32.5055     | 98.7108         | 98.7301       |
+| 0.6331        | 0.26  | 100  | 0.7266          | 0.0018         | -0.0892          | 0.5453             | 0.0910          | -36.0939       | -32.4406     | 98.7381         | 98.7510       |
+| 0.6048        | 0.52  | 200  | 0.7483          | -0.1575        | -0.2719          | 0.5282             | 0.1144          | -36.3550       | -32.6682     | 98.6127         | 98.6356       |
+| 0.5829        | 0.78  | 300  | 0.7391          | -0.0991        | -0.2198          | 0.5486             | 0.1207          | -36.2805       | -32.5848     | 98.6399         | 98.6637       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:984640c920b76b8731c289b4893327c5b19fa65b4236e746014aebd3c8e074fe
+oid sha256:cad0022958cf0d5deab2679b7475f2bb29807da71ff8617c795ec64a633d35b3
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.72113800048828,
-    "eval_logits/rejected": 98.69998931884766,
-    "eval_logps/chosen": -32.468719482421875,
-    "eval_logps/rejected": -36.04634094238281,
-    "eval_loss": 1.107681393623352,
-    "eval_rewards/accuracies": 0.4987541437149048,
-    "eval_rewards/chosen": -0.022987432777881622,
-    "eval_rewards/margins": 0.04884451627731323,
-    "eval_rewards/rejected": -0.07183194905519485,
-    "eval_runtime": 103.7689,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.305,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.7878170496457583,
-    "train_runtime": 2553.4781,
+    "train_loss": 0.5948986065852178,
+    "train_runtime": 2557.7017,
     "train_samples": 3079,
-    "train_samples_per_second": 1.206,
+    "train_samples_per_second": 1.204,
     "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.7878170496457583,
-    "train_runtime": 2553.4781,
+    "train_loss": 0.5948986065852178,
+    "train_runtime": 2557.7017,
     "train_samples": 3079,
-    "train_samples_per_second": 1.206,
+    "train_samples_per_second": 1.204,
     "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 118.0,
+      "grad_norm": 45.75,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
       "logps/chosen": -29.073104858398438,
       "logps/rejected": -26.25731658935547,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,629 +25,629 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 97.0,
+      "grad_norm": 38.75,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.08590698242188,
-      "logits/rejected": 80.79015350341797,
-      "logps/chosen": -34.172550201416016,
-      "logps/rejected": -32.99056625366211,
-      "loss": 0.9305,
-      "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.062255993485450745,
-      "rewards/margins": 0.09120028465986252,
-      "rewards/rejected": -0.028944293037056923,
+      "logits/chosen": 81.08214569091797,
+      "logits/rejected": 80.78972625732422,
+      "logps/chosen": -34.26863098144531,
+      "logps/rejected": -33.00303649902344,
+      "loss": 0.7238,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": -0.018833572044968605,
+      "rewards/margins": 0.012407698668539524,
+      "rewards/rejected": -0.03124127723276615,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 83.5,
+      "grad_norm": 37.25,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.69185638427734,
-      "logits/rejected": 80.58460235595703,
-      "logps/chosen": -33.645851135253906,
-      "logps/rejected": -30.73211097717285,
-      "loss": 1.0725,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.039649851620197296,
-      "rewards/margins": -0.0011898368829861283,
-      "rewards/rejected": 0.04083969444036484,
+      "logits/chosen": 80.68824005126953,
+      "logits/rejected": 80.57817840576172,
+      "logps/chosen": -33.58771514892578,
+      "logps/rejected": -30.75152015686035,
+      "loss": 0.7285,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0715368390083313,
+      "rewards/margins": 0.05335939675569534,
+      "rewards/rejected": 0.01817743293941021,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 105.5,
+      "grad_norm": 39.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.52629089355469,
-      "logits/rejected": 82.55845642089844,
-      "logps/chosen": -33.839778900146484,
-      "logps/rejected": -31.187374114990234,
-      "loss": 1.1222,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.15573647618293762,
-      "rewards/margins": -0.05707378312945366,
-      "rewards/rejected": 0.21281024813652039,
+      "logits/chosen": 82.5134506225586,
+      "logits/rejected": 82.5453872680664,
+      "logps/chosen": -33.79930877685547,
+      "logps/rejected": -31.215984344482422,
+      "loss": 0.7655,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.14945873618125916,
+      "rewards/margins": 0.00396394869312644,
+      "rewards/rejected": 0.14549477398395538,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 83.5,
+      "grad_norm": 44.5,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.00019073486328,
-      "logits/rejected": 80.99311828613281,
-      "logps/chosen": -32.78485870361328,
-      "logps/rejected": -33.173404693603516,
-      "loss": 1.023,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.35028138756752014,
-      "rewards/margins": 0.17467467486858368,
-      "rewards/rejected": 0.17560675740242004,
+      "logits/chosen": 81.0338363647461,
+      "logits/rejected": 81.03011322021484,
+      "logps/chosen": -32.87316131591797,
+      "logps/rejected": -33.17707061767578,
+      "loss": 0.7622,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.21063438057899475,
+      "rewards/margins": 0.07661890983581543,
+      "rewards/rejected": 0.13401541113853455,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 72.5,
+      "grad_norm": 29.125,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.55744934082031,
-      "logits/rejected": 78.57440948486328,
-      "logps/chosen": -30.58083724975586,
-      "logps/rejected": -30.754711151123047,
-      "loss": 1.0219,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.5619677901268005,
-      "rewards/margins": 0.2752513289451599,
-      "rewards/rejected": 0.2867165207862854,
+      "logits/chosen": 78.6342544555664,
+      "logits/rejected": 78.64932250976562,
+      "logps/chosen": -30.660537719726562,
+      "logps/rejected": -30.76174545288086,
+      "loss": 0.7328,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.38129910826683044,
+      "rewards/margins": 0.16322235763072968,
+      "rewards/rejected": 0.21807675063610077,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 94.5,
+      "grad_norm": 38.0,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.01163482666016,
-      "logits/rejected": 83.06632995605469,
-      "logps/chosen": -30.960674285888672,
-      "logps/rejected": -29.359949111938477,
-      "loss": 1.1647,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.19303548336029053,
-      "rewards/margins": -0.014480452053248882,
-      "rewards/rejected": 0.20751595497131348,
+      "logits/chosen": 83.13832092285156,
+      "logits/rejected": 83.19276428222656,
+      "logps/chosen": -30.93692970275879,
+      "logps/rejected": -29.44403648376465,
+      "loss": 0.7486,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.16676156222820282,
+      "rewards/margins": 0.0642227977514267,
+      "rewards/rejected": 0.10253874957561493,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 116.5,
+      "grad_norm": 64.5,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 83.62989044189453,
-      "logits/rejected": 83.66007232666016,
-      "logps/chosen": -30.669225692749023,
-      "logps/rejected": -33.12618637084961,
-      "loss": 1.1523,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.03618011251091957,
-      "rewards/margins": 0.04344985634088516,
-      "rewards/rejected": -0.07962997257709503,
+      "logits/chosen": 83.7562255859375,
+      "logits/rejected": 83.78288269042969,
+      "logps/chosen": -30.605281829833984,
+      "logps/rejected": -33.032676696777344,
+      "loss": 0.7785,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.01661914773285389,
+      "rewards/margins": 0.013097524642944336,
+      "rewards/rejected": 0.003521624254062772,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 85.0,
+      "grad_norm": 42.5,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.133056640625,
-      "logits/rejected": 81.11943054199219,
-      "logps/chosen": -31.318958282470703,
-      "logps/rejected": -30.970218658447266,
-      "loss": 0.845,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.167384073138237,
-      "rewards/margins": 0.3586970567703247,
-      "rewards/rejected": -0.1913129985332489,
+      "logits/chosen": 81.37464904785156,
+      "logits/rejected": 81.36463165283203,
+      "logps/chosen": -31.443639755249023,
+      "logps/rejected": -30.998950958251953,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04291192442178726,
+      "rewards/margins": 0.211822509765625,
+      "rewards/rejected": -0.16891059279441833,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 109.0,
+      "grad_norm": 38.5,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 77.96097564697266,
-      "logits/rejected": 77.93636322021484,
-      "logps/chosen": -32.162574768066406,
-      "logps/rejected": -31.054067611694336,
-      "loss": 0.7906,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.4280422329902649,
-      "rewards/margins": 0.5203782916069031,
-      "rewards/rejected": -0.09233605861663818,
+      "logits/chosen": 78.1283187866211,
+      "logits/rejected": 78.10060119628906,
+      "logps/chosen": -32.54193878173828,
+      "logps/rejected": -31.2618408203125,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.06736886501312256,
+      "rewards/margins": 0.2846258580684662,
+      "rewards/rejected": -0.21725702285766602,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 68.5,
+      "grad_norm": 33.0,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.30989837646484,
-      "logits/rejected": 83.33601379394531,
-      "logps/chosen": -33.85996627807617,
-      "logps/rejected": -31.64011573791504,
-      "loss": 0.9852,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.4005257189273834,
-      "rewards/margins": 0.30113959312438965,
-      "rewards/rejected": 0.09938610345125198,
+      "logits/chosen": 83.37786865234375,
+      "logits/rejected": 83.40235900878906,
+      "logps/chosen": -34.06679153442383,
+      "logps/rejected": -31.954029083251953,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.166738823056221,
+      "rewards/margins": 0.30917495489120483,
+      "rewards/rejected": -0.14243611693382263,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.69417572021484,
-      "eval_logits/rejected": 98.68303680419922,
-      "eval_logps/chosen": -32.46882629394531,
-      "eval_logps/rejected": -36.016597747802734,
-      "eval_loss": 1.1082578897476196,
-      "eval_rewards/accuracies": 0.4875415563583374,
-      "eval_rewards/chosen": -0.02308560535311699,
-      "eval_rewards/margins": 0.021978026255965233,
-      "eval_rewards/rejected": -0.04506362974643707,
-      "eval_runtime": 104.3924,
-      "eval_samples_per_second": 3.286,
-      "eval_steps_per_second": 0.412,
+      "eval_logits/chosen": 98.75102233886719,
+      "eval_logits/rejected": 98.73809814453125,
+      "eval_logps/chosen": -32.44057846069336,
+      "eval_logps/rejected": -36.09393310546875,
+      "eval_loss": 0.7266324758529663,
+      "eval_rewards/accuracies": 0.545265793800354,
+      "eval_rewards/chosen": 0.0018192834686487913,
+      "eval_rewards/margins": 0.09100572764873505,
+      "eval_rewards/rejected": -0.08918644487857819,
+      "eval_runtime": 104.1233,
+      "eval_samples_per_second": 3.294,
+      "eval_steps_per_second": 0.413,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 100.0,
+      "grad_norm": 53.25,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.43294525146484,
-      "logits/rejected": 83.3353042602539,
-      "logps/chosen": -32.524871826171875,
-      "logps/rejected": -32.74653625488281,
-      "loss": 0.8192,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4481372833251953,
-      "rewards/margins": 0.5297205448150635,
-      "rewards/rejected": -0.08158326148986816,
+      "logits/chosen": 83.52274322509766,
+      "logits/rejected": 83.4115219116211,
+      "logps/chosen": -32.51097869873047,
+      "logps/rejected": -32.80630874633789,
+      "loss": 0.6201,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.35827895998954773,
+      "rewards/margins": 0.46357136964797974,
+      "rewards/rejected": -0.10529237985610962,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 86.5,
+      "grad_norm": 50.25,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.61913299560547,
-      "logits/rejected": 83.73504638671875,
-      "logps/chosen": -28.402780532836914,
-      "logps/rejected": -35.50286865234375,
-      "loss": 0.7691,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.4741190969944,
-      "rewards/margins": 0.5134681463241577,
-      "rewards/rejected": -0.039348993450403214,
+      "logits/chosen": 83.7201919555664,
+      "logits/rejected": 83.82737731933594,
+      "logps/chosen": -28.233470916748047,
+      "logps/rejected": -35.50123977661133,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.4872798025608063,
+      "rewards/margins": 0.5167454481124878,
+      "rewards/rejected": -0.029465626925230026,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 72.0,
+      "grad_norm": 26.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 80.9137954711914,
-      "logits/rejected": 80.93389892578125,
-      "logps/chosen": -30.521535873413086,
-      "logps/rejected": -32.07093048095703,
-      "loss": 0.815,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.34723615646362305,
-      "rewards/margins": 0.47260579466819763,
-      "rewards/rejected": -0.12536963820457458,
+      "logits/chosen": 80.89537048339844,
+      "logits/rejected": 80.91288757324219,
+      "logps/chosen": -30.439437866210938,
+      "logps/rejected": -32.11792755126953,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.32754257321357727,
+      "rewards/margins": 0.457952082157135,
+      "rewards/rejected": -0.13040950894355774,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 86.0,
+      "grad_norm": 30.5,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.1353530883789,
-      "logits/rejected": 82.16053771972656,
-      "logps/chosen": -27.094701766967773,
-      "logps/rejected": -32.902889251708984,
-      "loss": 0.7235,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.3173602223396301,
-      "rewards/margins": 0.7489217519760132,
-      "rewards/rejected": -0.4315616488456726,
+      "logits/chosen": 82.11260223388672,
+      "logits/rejected": 82.12245178222656,
+      "logps/chosen": -27.101327896118164,
+      "logps/rejected": -33.005577087402344,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.24219810962677002,
+      "rewards/margins": 0.6497381329536438,
+      "rewards/rejected": -0.4075400233268738,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 59.75,
+      "grad_norm": 28.25,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.8051986694336,
-      "logits/rejected": 80.77262115478516,
-      "logps/chosen": -28.932445526123047,
-      "logps/rejected": -33.024757385253906,
-      "loss": 0.633,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4109979569911957,
-      "rewards/margins": 0.8234249949455261,
-      "rewards/rejected": -0.4124270975589752,
+      "logits/chosen": 80.61543273925781,
+      "logits/rejected": 80.58251953125,
+      "logps/chosen": -28.909435272216797,
+      "logps/rejected": -33.041297912597656,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3357718586921692,
+      "rewards/margins": 0.6681298613548279,
+      "rewards/rejected": -0.3323580324649811,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 82.5,
+      "grad_norm": 55.5,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 82.64894104003906,
-      "logits/rejected": 82.64643859863281,
-      "logps/chosen": -33.55781936645508,
-      "logps/rejected": -30.31070899963379,
-      "loss": 0.7859,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.5210397839546204,
-      "rewards/margins": 0.8433935046195984,
-      "rewards/rejected": -0.3223537802696228,
+      "logits/chosen": 82.49334716796875,
+      "logits/rejected": 82.49332427978516,
+      "logps/chosen": -33.531585693359375,
+      "logps/rejected": -30.385196685791016,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4236120283603668,
+      "rewards/margins": 0.7264719009399414,
+      "rewards/rejected": -0.3028598725795746,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 81.5,
+      "grad_norm": 42.25,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 83.43087005615234,
-      "logits/rejected": 83.37384033203125,
-      "logps/chosen": -30.71698570251465,
-      "logps/rejected": -32.415626525878906,
-      "loss": 0.6671,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4880717396736145,
-      "rewards/margins": 0.8440794944763184,
-      "rewards/rejected": -0.35600775480270386,
+      "logits/chosen": 83.25149536132812,
+      "logits/rejected": 83.19024658203125,
+      "logps/chosen": -30.89450454711914,
+      "logps/rejected": -32.51388931274414,
+      "loss": 0.5987,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2553495168685913,
+      "rewards/margins": 0.601028323173523,
+      "rewards/rejected": -0.3456788957118988,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 64.0,
+      "grad_norm": 34.25,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 81.02064514160156,
-      "logits/rejected": 81.00648498535156,
-      "logps/chosen": -30.46847152709961,
-      "logps/rejected": -31.655284881591797,
-      "loss": 0.6529,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.6558108925819397,
-      "rewards/margins": 1.104081153869629,
-      "rewards/rejected": -0.4482702314853668,
+      "logits/chosen": 80.78834533691406,
+      "logits/rejected": 80.77064514160156,
+      "logps/chosen": -30.47861671447754,
+      "logps/rejected": -31.64987564086914,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.5029744505882263,
+      "rewards/margins": 0.8478416204452515,
+      "rewards/rejected": -0.34486719965934753,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 27.25,
+      "grad_norm": 21.125,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 82.70450592041016,
-      "logits/rejected": 82.70726776123047,
-      "logps/chosen": -30.44081687927246,
-      "logps/rejected": -30.764562606811523,
-      "loss": 0.8467,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2930375039577484,
-      "rewards/margins": 0.5599225759506226,
-      "rewards/rejected": -0.2668851315975189,
+      "logits/chosen": 82.49182891845703,
+      "logits/rejected": 82.4795150756836,
+      "logps/chosen": -30.340301513671875,
+      "logps/rejected": -30.779190063476562,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2982791066169739,
+      "rewards/margins": 0.5160930752754211,
+      "rewards/rejected": -0.21781396865844727,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 68.0,
+      "grad_norm": 17.375,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 78.14289093017578,
-      "logits/rejected": 78.0774917602539,
-      "logps/chosen": -34.14347839355469,
-      "logps/rejected": -32.717750549316406,
-      "loss": 0.902,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.5483377575874329,
-      "rewards/margins": 0.7502498030662537,
-      "rewards/rejected": -0.2019120752811432,
+      "logits/chosen": 77.9924545288086,
+      "logits/rejected": 77.93614196777344,
+      "logps/chosen": -33.81483459472656,
+      "logps/rejected": -32.65379333496094,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.6565331816673279,
+      "rewards/margins": 0.7688080072402954,
+      "rewards/rejected": -0.1122748851776123,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.65617370605469,
-      "eval_logits/rejected": 98.62713623046875,
-      "eval_logps/chosen": -32.60585021972656,
-      "eval_logps/rejected": -36.384361267089844,
-      "eval_loss": 0.9846288561820984,
-      "eval_rewards/accuracies": 0.554817259311676,
-      "eval_rewards/chosen": -0.14640627801418304,
-      "eval_rewards/margins": 0.22964100539684296,
-      "eval_rewards/rejected": -0.3760472238063812,
-      "eval_runtime": 103.9231,
-      "eval_samples_per_second": 3.301,
-      "eval_steps_per_second": 0.414,
+      "eval_logits/chosen": 98.63562774658203,
+      "eval_logits/rejected": 98.61270141601562,
+      "eval_logps/chosen": -32.66818618774414,
+      "eval_logps/rejected": -36.3549919128418,
+      "eval_loss": 0.7483024001121521,
+      "eval_rewards/accuracies": 0.5282392501831055,
+      "eval_rewards/chosen": -0.15750552713871002,
+      "eval_rewards/margins": 0.1144195944070816,
+      "eval_rewards/rejected": -0.27192509174346924,
+      "eval_runtime": 104.0056,
+      "eval_samples_per_second": 3.298,
+      "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 107.0,
+      "grad_norm": 67.5,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 80.76277160644531,
-      "logits/rejected": 80.67192840576172,
-      "logps/chosen": -33.424560546875,
-      "logps/rejected": -35.27748107910156,
-      "loss": 0.7859,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4690447747707367,
-      "rewards/margins": 0.7312911152839661,
-      "rewards/rejected": -0.26224634051322937,
+      "logits/chosen": 80.62068176269531,
+      "logits/rejected": 80.52841186523438,
+      "logps/chosen": -33.23737716674805,
+      "logps/rejected": -35.3394889831543,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.49584078788757324,
+      "rewards/margins": 0.7432178258895874,
+      "rewards/rejected": -0.2473769634962082,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 86.0,
+      "grad_norm": 22.875,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 82.9009017944336,
-      "logits/rejected": 82.97982788085938,
-      "logps/chosen": -31.129741668701172,
-      "logps/rejected": -31.022235870361328,
-      "loss": 0.566,
+      "logits/chosen": 82.79103088378906,
+      "logits/rejected": 82.86891174316406,
+      "logps/chosen": -31.00775718688965,
+      "logps/rejected": -31.1812801361084,
+      "loss": 0.4699,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.7372905611991882,
-      "rewards/margins": 1.001245141029358,
-      "rewards/rejected": -0.2639545798301697,
+      "rewards/chosen": 0.6588366627693176,
+      "rewards/margins": 0.9754641652107239,
+      "rewards/rejected": -0.31662750244140625,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 79.0,
+      "grad_norm": 39.75,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 80.10737609863281,
-      "logits/rejected": 80.16769409179688,
-      "logps/chosen": -32.2999153137207,
-      "logps/rejected": -34.19389343261719,
-      "loss": 0.904,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.45524734258651733,
-      "rewards/margins": 0.6188174486160278,
-      "rewards/rejected": -0.16357013583183289,
+      "logits/chosen": 79.89860534667969,
+      "logits/rejected": 79.95353698730469,
+      "logps/chosen": -32.31645965576172,
+      "logps/rejected": -34.39720153808594,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3425000309944153,
+      "rewards/margins": 0.612037718296051,
+      "rewards/rejected": -0.26953771710395813,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 101.0,
+      "grad_norm": 34.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 82.58370208740234,
-      "logits/rejected": 82.85044860839844,
-      "logps/chosen": -30.799701690673828,
-      "logps/rejected": -31.604589462280273,
-      "loss": 0.7596,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7995287179946899,
-      "rewards/margins": 0.8648217916488647,
-      "rewards/rejected": -0.06529306620359421,
+      "logits/chosen": 82.30177307128906,
+      "logits/rejected": 82.58096313476562,
+      "logps/chosen": -30.619409561157227,
+      "logps/rejected": -31.930099487304688,
+      "loss": 0.4503,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7480591535568237,
+      "rewards/margins": 1.0266973972320557,
+      "rewards/rejected": -0.2786383032798767,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 89.5,
+      "grad_norm": 37.25,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 81.37136840820312,
-      "logits/rejected": 81.4355697631836,
-      "logps/chosen": -26.921367645263672,
-      "logps/rejected": -30.0640811920166,
-      "loss": 0.8158,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.6070747971534729,
-      "rewards/margins": 0.754082202911377,
-      "rewards/rejected": -0.14700737595558167,
+      "logits/chosen": 81.00114440917969,
+      "logits/rejected": 81.05775451660156,
+      "logps/chosen": -26.927043914794922,
+      "logps/rejected": -30.175378799438477,
+      "loss": 0.5818,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.46819525957107544,
+      "rewards/margins": 0.6604421138763428,
+      "rewards/rejected": -0.19224683940410614,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 69.0,
+      "grad_norm": 28.125,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 78.6139144897461,
-      "logits/rejected": 78.74816131591797,
-      "logps/chosen": -30.19488525390625,
-      "logps/rejected": -36.41002655029297,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.2295000553131104,
-      "rewards/margins": 1.5070557594299316,
-      "rewards/rejected": -0.2775557041168213,
+      "logits/chosen": 78.21713256835938,
+      "logits/rejected": 78.345458984375,
+      "logps/chosen": -30.480411529541016,
+      "logps/rejected": -36.508689880371094,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.7564077377319336,
+      "rewards/margins": 1.0413528680801392,
+      "rewards/rejected": -0.28494516015052795,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 36.0,
+      "grad_norm": 24.875,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 78.08236694335938,
-      "logits/rejected": 78.10992431640625,
-      "logps/chosen": -30.87982749938965,
-      "logps/rejected": -31.692874908447266,
-      "loss": 0.6864,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.8091908693313599,
-      "rewards/margins": 0.977896511554718,
-      "rewards/rejected": -0.16870568692684174,
+      "logits/chosen": 77.48748779296875,
+      "logits/rejected": 77.51399230957031,
+      "logps/chosen": -30.899953842163086,
+      "logps/rejected": -31.809417724609375,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.6152798533439636,
+      "rewards/margins": 0.828079104423523,
+      "rewards/rejected": -0.2127993404865265,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 113.0,
+      "grad_norm": 40.5,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 80.7326431274414,
-      "logits/rejected": 80.51844024658203,
-      "logps/chosen": -31.088062286376953,
-      "logps/rejected": -29.66598892211914,
-      "loss": 0.9291,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.6496511697769165,
-      "rewards/margins": 0.6487425565719604,
-      "rewards/rejected": 0.0009086370700970292,
+      "logits/chosen": 80.2722396850586,
+      "logits/rejected": 80.06110382080078,
+      "logps/chosen": -31.229726791381836,
+      "logps/rejected": -29.85305404663086,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.406115859746933,
+      "rewards/margins": 0.5363569259643555,
+      "rewards/rejected": -0.1302410513162613,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 72.0,
+      "grad_norm": 23.5,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 80.9128189086914,
-      "logits/rejected": 80.82886505126953,
-      "logps/chosen": -33.01869201660156,
-      "logps/rejected": -32.421485900878906,
-      "loss": 0.5234,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.977002739906311,
-      "rewards/margins": 1.3411604166030884,
-      "rewards/rejected": -0.3641577661037445,
+      "logits/chosen": 80.33818054199219,
+      "logits/rejected": 80.25775146484375,
+      "logps/chosen": -33.049842834472656,
+      "logps/rejected": -32.65058135986328,
+      "loss": 0.4554,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7380836606025696,
+      "rewards/margins": 1.1816825866699219,
+      "rewards/rejected": -0.44359898567199707,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 70.5,
+      "grad_norm": 40.0,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 76.50440979003906,
-      "logits/rejected": 76.59843444824219,
-      "logps/chosen": -32.13983917236328,
-      "logps/rejected": -29.128122329711914,
-      "loss": 0.671,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.139281988143921,
-      "rewards/margins": 1.1627700328826904,
-      "rewards/rejected": -0.023488014936447144,
+      "logits/chosen": 76.02481079101562,
+      "logits/rejected": 76.12067413330078,
+      "logps/chosen": -32.21509552001953,
+      "logps/rejected": -29.180316925048828,
+      "loss": 0.5829,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.8334287405014038,
+      "rewards/margins": 0.8882354497909546,
+      "rewards/rejected": -0.054806679487228394,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 98.7301254272461,
-      "eval_logits/rejected": 98.71080017089844,
-      "eval_logps/chosen": -32.50548553466797,
-      "eval_logps/rejected": -36.0523567199707,
-      "eval_loss": 1.1081271171569824,
-      "eval_rewards/accuracies": 0.47757473587989807,
-      "eval_rewards/chosen": -0.056078068912029266,
-      "eval_rewards/margins": 0.021167948842048645,
-      "eval_rewards/rejected": -0.07724600285291672,
-      "eval_runtime": 103.8638,
-      "eval_samples_per_second": 3.302,
+      "eval_logits/chosen": 98.6636962890625,
+      "eval_logits/rejected": 98.63994598388672,
+      "eval_logps/chosen": -32.58477020263672,
+      "eval_logps/rejected": -36.28050231933594,
+      "eval_loss": 0.7390850782394409,
+      "eval_rewards/accuracies": 0.5485880374908447,
+      "eval_rewards/chosen": -0.09911961853504181,
+      "eval_rewards/margins": 0.12066645920276642,
+      "eval_rewards/rejected": -0.21978609263896942,
+      "eval_runtime": 103.9194,
+      "eval_samples_per_second": 3.301,
       "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 66.0,
+      "grad_norm": 31.5,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 83.55355072021484,
-      "logits/rejected": 83.5801773071289,
-      "logps/chosen": -30.023509979248047,
-      "logps/rejected": -32.43012237548828,
-      "loss": 0.6531,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7560194730758667,
-      "rewards/margins": 0.9633440971374512,
-      "rewards/rejected": -0.20732466876506805,
+      "logits/chosen": 83.141357421875,
+      "logits/rejected": 83.17015075683594,
+      "logps/chosen": -30.017724990844727,
+      "logps/rejected": -32.537620544433594,
+      "loss": 0.5115,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.5920647382736206,
+      "rewards/margins": 0.8285657167434692,
+      "rewards/rejected": -0.23650094866752625,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 56.25,
+      "grad_norm": 25.5,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 81.07228088378906,
-      "logits/rejected": 81.07295989990234,
-      "logps/chosen": -30.569133758544922,
-      "logps/rejected": -29.126922607421875,
-      "loss": 0.5552,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.926487147808075,
-      "rewards/margins": 1.1131722927093506,
-      "rewards/rejected": -0.1866852045059204,
+      "logits/chosen": 80.59849548339844,
+      "logits/rejected": 80.60069274902344,
+      "logps/chosen": -30.53042221069336,
+      "logps/rejected": -29.161365509033203,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.7476966977119446,
+      "rewards/margins": 0.9170078039169312,
+      "rewards/rejected": -0.16931119561195374,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 65.5,
+      "grad_norm": 22.25,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 78.3108139038086,
-      "logits/rejected": 78.35076904296875,
-      "logps/chosen": -29.139026641845703,
-      "logps/rejected": -32.85981750488281,
-      "loss": 0.5897,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 1.0922209024429321,
-      "rewards/margins": 1.294679045677185,
-      "rewards/rejected": -0.20245835185050964,
+      "logits/chosen": 77.74131774902344,
+      "logits/rejected": 77.79161071777344,
+      "logps/chosen": -29.038299560546875,
+      "logps/rejected": -32.908966064453125,
+      "loss": 0.458,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.9200155138969421,
+      "rewards/margins": 1.111884355545044,
+      "rewards/rejected": -0.19186890125274658,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 98.5,
+      "grad_norm": 57.25,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 82.51289367675781,
-      "logits/rejected": 82.52921295166016,
-      "logps/chosen": -32.200294494628906,
-      "logps/rejected": -33.6099853515625,
-      "loss": 0.7511,
+      "logits/chosen": 82.08992767333984,
+      "logits/rejected": 82.12026977539062,
+      "logps/chosen": -32.29141616821289,
+      "logps/rejected": -33.880916595458984,
+      "loss": 0.5523,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.910667896270752,
-      "rewards/margins": 1.127828598022461,
-      "rewards/rejected": -0.21716061234474182,
+      "rewards/chosen": 0.6445139050483704,
+      "rewards/margins": 1.003070592880249,
+      "rewards/rejected": -0.3585566580295563,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 47.0,
+      "grad_norm": 14.125,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 81.58796691894531,
-      "logits/rejected": 81.5913314819336,
-      "logps/chosen": -32.477813720703125,
-      "logps/rejected": -33.15494155883789,
-      "loss": 0.6388,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 1.049329400062561,
-      "rewards/margins": 1.106154203414917,
-      "rewards/rejected": -0.056824732571840286,
+      "logits/chosen": 81.13373565673828,
+      "logits/rejected": 81.14064025878906,
+      "logps/chosen": -32.35675048828125,
+      "logps/rejected": -33.414161682128906,
+      "loss": 0.4904,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9008921384811401,
+      "rewards/margins": 1.1265466213226318,
+      "rewards/rejected": -0.22565443813800812,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 59.0,
+      "grad_norm": 27.625,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 83.0643539428711,
-      "logits/rejected": 83.09364318847656,
-      "logps/chosen": -28.514789581298828,
-      "logps/rejected": -31.751323699951172,
-      "loss": 0.5731,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.9336811304092407,
-      "rewards/margins": 1.0250084400177002,
-      "rewards/rejected": -0.09132737666368484,
+      "logits/chosen": 82.62144470214844,
+      "logits/rejected": 82.64656066894531,
+      "logps/chosen": -28.411449432373047,
+      "logps/rejected": -31.78824806213379,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.7985339760780334,
+      "rewards/margins": 0.8954124450683594,
+      "rewards/rejected": -0.09687861800193787,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 121.5,
+      "grad_norm": 43.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 82.48683166503906,
-      "logits/rejected": 82.50460815429688,
-      "logps/chosen": -31.874202728271484,
-      "logps/rejected": -35.20905303955078,
-      "loss": 0.7031,
+      "logits/chosen": 82.0575942993164,
+      "logits/rejected": 82.07881164550781,
+      "logps/chosen": -31.82853126525879,
+      "logps/rejected": -35.34919357299805,
+      "loss": 0.6002,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.9017645120620728,
-      "rewards/margins": 0.9914749264717102,
-      "rewards/rejected": -0.08971036225557327,
+      "rewards/chosen": 0.7333434820175171,
+      "rewards/margins": 0.9012172818183899,
+      "rewards/rejected": -0.1678738296031952,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 94.5,
+      "grad_norm": 34.75,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 76.47808074951172,
-      "logits/rejected": 76.34886932373047,
-      "logps/chosen": -29.865795135498047,
-      "logps/rejected": -28.233165740966797,
-      "loss": 0.7695,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.6392983794212341,
-      "rewards/margins": 0.7046308517456055,
-      "rewards/rejected": -0.06533239781856537,
+      "logits/chosen": 75.95097351074219,
+      "logits/rejected": 75.82870483398438,
+      "logps/chosen": -29.8321475982666,
+      "logps/rejected": -28.438806533813477,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5207866430282593,
+      "rewards/margins": 0.7155483365058899,
+      "rewards/rejected": -0.19476178288459778,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.7878170496457583,
-      "train_runtime": 2553.4781,
-      "train_samples_per_second": 1.206,
+      "train_loss": 0.5948986065852178,
+      "train_runtime": 2557.7017,
+      "train_samples_per_second": 1.204,
       "train_steps_per_second": 0.151
     }
   ],