Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2457.5789
-- Rewards/chosen: -0.0084
-- Rewards/rejected: -0.0130
-- Rewards/accuracies: 0.5893
-- Rewards/margins: 0.0047
-- Logps/rejected: -37.2694
-- Logps/chosen: -33.2802
-- Logits/rejected: 97.7521
-- Logits/chosen: 97.7850
+- Loss: 1.5174
+- Rewards/chosen: 0.1203
+- Rewards/rejected: 0.1230
+- Rewards/accuracies: 0.5108
+- Rewards/margins: -0.0027
+- Logps/rejected: -35.8299
+- Logps/chosen: -32.3095
+- Logits/rejected: 99.0568
+- Logits/chosen: 99.0617
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2452.7488     | 0.26  | 100  | 2493.3496       | 0.0014         | 0.0005           | 0.5282             | 0.0008          | -35.9144       | -32.3070     | 98.7114         | 98.7186       |
-| 2352.7865     | 0.52  | 200  | 2463.5779       | -0.0049        | -0.0088          | 0.5777             | 0.0039          | -36.8483       | -32.9300     | 98.0442         | 98.0668       |
-| 2331.667      | 0.78  | 300  | 2455.6594       | -0.0084        | -0.0132          | 0.5772             | 0.0049          | -37.2893       | -33.2808     | 97.7712         | 97.8048       |
+| 2.0775        | 0.26  | 100  | 1.7337          | -0.0836        | -0.0209          | 0.4747             | -0.0626         | -35.9898       | -32.5360     | 98.9624         | 98.9691       |
+| 3.1221        | 0.52  | 200  | 1.7165          | -0.0706        | -0.1245          | 0.5278             | 0.0539          | -36.1048       | -32.5217     | 99.1580         | 99.1691       |
+| 2.4404        | 0.78  | 300  | 1.5174          | 0.1203         | 0.1230           | 0.5108             | -0.0027         | -35.8299       | -32.3095     | 99.0568         | 99.0617       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6577b57be00a7e926370a1a2670f22ad1194b5df9ab22cbfdbe2fe42a7631c53
+oid sha256:43930e31eed30f192e67c6779e4f3c9f5842af7c56ff7c216a985f72af451378
 size 671150064

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 97.7850112915039,
-    "eval_logits/rejected": 97.75206756591797,
-    "eval_logps/chosen": -33.28022384643555,
-    "eval_logps/rejected": -37.269412994384766,
-    "eval_loss": 2457.578857421875,
-    "eval_rewards/accuracies": 0.5892857313156128,
-    "eval_rewards/chosen": -0.008370478637516499,
-    "eval_rewards/margins": 0.004658386576920748,
-    "eval_rewards/rejected": -0.013028863817453384,
-    "eval_runtime": 103.6308,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.31,
-    "eval_steps_per_second": 0.415,
-    "train_loss": 2389.4745231331167,
-    "train_runtime": 2559.5069,
+    "train_loss": 2.2499145250041765,
+    "train_runtime": 2558.7273,
     "train_samples": 3079,
     "train_samples_per_second": 1.203,
     "train_steps_per_second": 0.15

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 2389.4745231331167,
-    "train_runtime": 2559.5069,
+    "train_loss": 2.2499145250041765,
+    "train_runtime": 2558.7273,
     "train_samples": 3079,
     "train_samples_per_second": 1.203,
     "train_steps_per_second": 0.15

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 13056.0,
+      "grad_norm": 145.0,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
       "logps/chosen": -29.073104858398438,
       "logps/rejected": -26.25731658935547,
-      "loss": 2500.0,
+      "loss": 0.3086,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,628 +25,628 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 10816.0,
+      "grad_norm": 288.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.07842254638672,
-      "logits/rejected": 80.78025817871094,
-      "logps/chosen": -34.27768325805664,
-      "logps/rejected": -32.99871063232422,
-      "loss": 2500.2613,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": -0.0003595556481741369,
-      "rewards/margins": 4.3542200728552416e-05,
-      "rewards/rejected": -0.0004030978016089648,
+      "logits/chosen": 81.08262634277344,
+      "logits/rejected": 80.7869873046875,
+      "logps/chosen": -34.28562545776367,
+      "logps/rejected": -33.03427505493164,
+      "loss": 0.9415,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": -0.039508190006017685,
+      "rewards/margins": 0.02877185121178627,
+      "rewards/rejected": -0.06828003376722336,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 12480.0,
+      "grad_norm": 149.0,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.66203308105469,
-      "logits/rejected": 80.55176544189453,
-      "logps/chosen": -33.65496826171875,
-      "logps/rejected": -30.774953842163086,
-      "loss": 2497.7465,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.00034939637407660484,
-      "rewards/margins": 0.0003240474034100771,
-      "rewards/rejected": 2.534888881200459e-05,
+      "logits/chosen": 80.65168762207031,
+      "logits/rejected": 80.53875732421875,
+      "logps/chosen": -33.57862091064453,
+      "logps/rejected": -30.82345199584961,
+      "loss": 1.1289,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.1001572236418724,
+      "rewards/margins": 0.14152315258979797,
+      "rewards/rejected": -0.041365914046764374,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 12032.0,
+      "grad_norm": 234.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.52423095703125,
-      "logits/rejected": 82.56011199951172,
-      "logps/chosen": -33.83774185180664,
-      "logps/rejected": -31.181049346923828,
-      "loss": 2508.2553,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0017508044838905334,
-      "rewards/margins": -0.0006770493928343058,
-      "rewards/rejected": 0.002427854109555483,
+      "logits/chosen": 82.32148742675781,
+      "logits/rejected": 82.35160827636719,
+      "logps/chosen": -33.95701599121094,
+      "logps/rejected": -31.341405868530273,
+      "loss": 1.3887,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.05022481083869934,
+      "rewards/margins": -0.023957695811986923,
+      "rewards/rejected": 0.07418251037597656,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 11584.0,
+      "grad_norm": 636.0,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.07867431640625,
-      "logits/rejected": 81.0750503540039,
-      "logps/chosen": -32.68397521972656,
-      "logps/rejected": -33.1097412109375,
-      "loss": 2479.0717,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.004900880623608828,
-      "rewards/margins": 0.0023130704648792744,
-      "rewards/rejected": 0.002587809693068266,
+      "logits/chosen": 80.53231811523438,
+      "logits/rejected": 80.530029296875,
+      "logps/chosen": -33.09763717651367,
+      "logps/rejected": -33.38296890258789,
+      "loss": 2.1646,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06877875328063965,
+      "rewards/margins": 0.0817752406001091,
+      "rewards/rejected": -0.0129964929074049,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 10944.0,
+      "grad_norm": 386.0,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.82911682128906,
-      "logits/rejected": 78.838623046875,
-      "logps/chosen": -30.458852767944336,
-      "logps/rejected": -30.663738250732422,
-      "loss": 2470.3088,
+      "logits/chosen": 78.10733032226562,
+      "logits/rejected": 78.12706756591797,
+      "logps/chosen": -31.247085571289062,
+      "logps/rejected": -31.239765167236328,
+      "loss": 1.232,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.007463912479579449,
-      "rewards/margins": 0.0033684559166431427,
-      "rewards/rejected": 0.004095455631613731,
+      "rewards/chosen": -0.03765222057700157,
+      "rewards/margins": 0.11217772960662842,
+      "rewards/rejected": -0.14982998371124268,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 9792.0,
+      "grad_norm": 200.0,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.4775619506836,
-      "logits/rejected": 83.53484344482422,
-      "logps/chosen": -30.7824764251709,
-      "logps/rejected": -29.335651397705078,
-      "loss": 2489.6246,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.003926820587366819,
-      "rewards/margins": 0.0013781532179564238,
-      "rewards/rejected": 0.0025486673694103956,
+      "logits/chosen": 82.77879333496094,
+      "logits/rejected": 82.82901763916016,
+      "logps/chosen": -31.19879150390625,
+      "logps/rejected": -29.63169288635254,
+      "loss": 1.1982,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.021271925419569016,
+      "rewards/margins": 0.015784021466970444,
+      "rewards/rejected": -0.03705594688653946,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 12288.0,
+      "grad_norm": 832.0,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 84.08866119384766,
-      "logits/rejected": 84.1230697631836,
-      "logps/chosen": -30.227558135986328,
-      "logps/rejected": -32.61187744140625,
-      "loss": 2504.2023,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.004014655016362667,
-      "rewards/margins": -0.0002436217910144478,
-      "rewards/rejected": 0.004258277826011181,
+      "logits/chosen": 83.74064636230469,
+      "logits/rejected": 83.76899719238281,
+      "logps/chosen": -30.796112060546875,
+      "logps/rejected": -33.087310791015625,
+      "loss": 2.0455,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.15038001537322998,
+      "rewards/margins": -0.10573381185531616,
+      "rewards/rejected": -0.04464619606733322,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 12288.0,
+      "grad_norm": 432.0,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.83061218261719,
-      "logits/rejected": 81.81108093261719,
-      "logps/chosen": -30.975017547607422,
-      "logps/rejected": -30.6242733001709,
-      "loss": 2462.8395,
+      "logits/chosen": 81.53680419921875,
+      "logits/rejected": 81.53197479248047,
+      "logps/chosen": -31.50222396850586,
+      "logps/rejected": -30.938213348388672,
+      "loss": 2.2722,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.005299207754433155,
-      "rewards/margins": 0.0039654383435845375,
-      "rewards/rejected": 0.0013337695272639394,
+      "rewards/chosen": 0.0024462162982672453,
+      "rewards/margins": 0.16495725512504578,
+      "rewards/rejected": -0.16251103579998016,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 14336.0,
+      "grad_norm": 378.0,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.58903503417969,
-      "logits/rejected": 78.56108093261719,
-      "logps/chosen": -32.038856506347656,
-      "logps/rejected": -30.935333251953125,
-      "loss": 2445.0191,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.005993223749101162,
-      "rewards/margins": 0.005831850226968527,
-      "rewards/rejected": 0.0001613735657883808,
+      "logits/chosen": 78.6462173461914,
+      "logits/rejected": 78.61238098144531,
+      "logps/chosen": -32.54735565185547,
+      "logps/rejected": -30.98733139038086,
+      "loss": 1.7909,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0817408338189125,
+      "rewards/margins": 0.11401765048503876,
+      "rewards/rejected": -0.03227682039141655,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 12480.0,
+      "grad_norm": 210.0,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.63715362548828,
-      "logits/rejected": 83.66267395019531,
-      "logps/chosen": -33.82961654663086,
-      "logps/rejected": -31.767969131469727,
-      "loss": 2452.7488,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.004753750748932362,
-      "rewards/margins": 0.004927995149046183,
-      "rewards/rejected": -0.00017424393445253372,
+      "logits/chosen": 84.08260345458984,
+      "logits/rejected": 84.1148910522461,
+      "logps/chosen": -34.308895111083984,
+      "logps/rejected": -31.754663467407227,
+      "loss": 2.0775,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.003516471479088068,
+      "rewards/margins": 0.00018945932970382273,
+      "rewards/rejected": -0.0037059492897242308,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.71858215332031,
-      "eval_logits/rejected": 98.71143341064453,
-      "eval_logps/chosen": -32.30698013305664,
-      "eval_logps/rejected": -35.91435241699219,
-      "eval_loss": 2493.349609375,
-      "eval_rewards/accuracies": 0.5282392501831055,
-      "eval_rewards/chosen": 0.0013619456440210342,
-      "eval_rewards/margins": 0.0008401816012337804,
-      "eval_rewards/rejected": 0.000521763926371932,
-      "eval_runtime": 104.2304,
-      "eval_samples_per_second": 3.291,
+      "eval_logits/chosen": 98.9691390991211,
+      "eval_logits/rejected": 98.96244812011719,
+      "eval_logps/chosen": -32.53602981567383,
+      "eval_logps/rejected": -35.98977279663086,
+      "eval_loss": 1.7336534261703491,
+      "eval_rewards/accuracies": 0.4746677577495575,
+      "eval_rewards/chosen": -0.08356913179159164,
+      "eval_rewards/margins": -0.06264925748109818,
+      "eval_rewards/rejected": -0.02091986872255802,
+      "eval_runtime": 104.1909,
+      "eval_samples_per_second": 3.292,
       "eval_steps_per_second": 0.413,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 14144.0,
+      "grad_norm": 256.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.81652069091797,
-      "logits/rejected": 83.7025146484375,
-      "logps/chosen": -32.34592819213867,
-      "logps/rejected": -32.730064392089844,
-      "loss": 2428.6338,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.006768748164176941,
-      "rewards/margins": 0.007510475814342499,
-      "rewards/rejected": -0.0007417276501655579,
+      "logits/chosen": 84.40672302246094,
+      "logits/rejected": 84.30145263671875,
+      "logps/chosen": -32.60901641845703,
+      "logps/rejected": -32.70885467529297,
+      "loss": 3.1161,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.37241029739379883,
+      "rewards/margins": 0.4200804829597473,
+      "rewards/rejected": -0.04767021909356117,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 13120.0,
+      "grad_norm": 446.0,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.8105239868164,
-      "logits/rejected": 83.92085266113281,
-      "logps/chosen": -28.203353881835938,
-      "logps/rejected": -35.42558288574219,
-      "loss": 2433.6672,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.007262272294610739,
-      "rewards/margins": 0.00692665483802557,
-      "rewards/rejected": 0.00033561658347025514,
+      "logits/chosen": 84.58103942871094,
+      "logits/rejected": 84.6691665649414,
+      "logps/chosen": -28.995365142822266,
+      "logps/rejected": -35.473243713378906,
+      "loss": 2.9665,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.05920146033167839,
+      "rewards/margins": -0.0465129017829895,
+      "rewards/rejected": -0.012688541784882545,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 9536.0,
+      "grad_norm": 208.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 80.93511199951172,
-      "logits/rejected": 80.96368408203125,
-      "logps/chosen": -30.244766235351562,
-      "logps/rejected": -31.988866806030273,
-      "loss": 2432.2412,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.006625907961279154,
-      "rewards/margins": 0.007198316045105457,
-      "rewards/rejected": -0.0005724082584492862,
+      "logits/chosen": 82.01698303222656,
+      "logits/rejected": 82.04023742675781,
+      "logps/chosen": -30.6518497467041,
+      "logps/rejected": -32.01683807373047,
+      "loss": 1.858,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.22995714843273163,
+      "rewards/margins": 0.3066490590572357,
+      "rewards/rejected": -0.0766918882727623,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 10752.0,
+      "grad_norm": 392.0,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 81.802734375,
-      "logits/rejected": 81.80996704101562,
-      "logps/chosen": -26.940113067626953,
-      "logps/rejected": -33.06999969482422,
-      "loss": 2390.4303,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.005072128027677536,
-      "rewards/margins": 0.011538348160684109,
-      "rewards/rejected": -0.006466220133006573,
+      "logits/chosen": 83.59390258789062,
+      "logits/rejected": 83.58688354492188,
+      "logps/chosen": -27.2325439453125,
+      "logps/rejected": -32.482357025146484,
+      "loss": 2.0491,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.19329962134361267,
+      "rewards/margins": 0.24638020992279053,
+      "rewards/rejected": -0.05308058112859726,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 11968.0,
+      "grad_norm": 209.0,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.1136474609375,
-      "logits/rejected": 80.08084106445312,
-      "logps/chosen": -28.865270614624023,
-      "logps/rejected": -33.2497444152832,
-      "loss": 2385.4119,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.005238414742052555,
-      "rewards/margins": 0.012070849537849426,
-      "rewards/rejected": -0.006832434795796871,
+      "logits/chosen": 82.43211364746094,
+      "logits/rejected": 82.41563415527344,
+      "logps/chosen": -28.861217498779297,
+      "logps/rejected": -32.51410675048828,
+      "loss": 1.804,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.4751041829586029,
+      "rewards/margins": 0.42794743180274963,
+      "rewards/rejected": 0.04715672880411148,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 13504.0,
+      "grad_norm": 450.0,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 81.72358703613281,
-      "logits/rejected": 81.7463607788086,
-      "logps/chosen": -34.06095886230469,
-      "logps/rejected": -30.822778701782227,
-      "loss": 2411.9418,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.0007578867371194065,
-      "rewards/margins": 0.009460283443331718,
-      "rewards/rejected": -0.008702396415174007,
+      "logits/chosen": 84.4583969116211,
+      "logits/rejected": 84.46671295166016,
+      "logps/chosen": -33.707855224609375,
+      "logps/rejected": -29.813602447509766,
+      "loss": 3.5567,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.38600218296051025,
+      "rewards/margins": 0.2609595060348511,
+      "rewards/rejected": 0.12504267692565918,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 14656.0,
+      "grad_norm": 278.0,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 82.34159851074219,
-      "logits/rejected": 82.29017639160156,
-      "logps/chosen": -31.006017684936523,
-      "logps/rejected": -33.12128829956055,
-      "loss": 2373.2635,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0025326875038444996,
-      "rewards/margins": 0.013544997200369835,
-      "rewards/rejected": -0.011012310162186623,
+      "logits/chosen": 85.22765350341797,
+      "logits/rejected": 85.16728973388672,
+      "logps/chosen": -31.05304527282715,
+      "logps/rejected": -32.11688995361328,
+      "loss": 1.9364,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.1856154203414917,
+      "rewards/margins": 0.2727610170841217,
+      "rewards/rejected": -0.08714555948972702,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 11584.0,
+      "grad_norm": 196.0,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 79.51490783691406,
-      "logits/rejected": 79.4914779663086,
-      "logps/chosen": -30.92569923400879,
-      "logps/rejected": -32.057472229003906,
-      "loss": 2389.5596,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.0027145310305058956,
-      "rewards/margins": 0.011717178858816624,
-      "rewards/rejected": -0.00900264736264944,
+      "logits/chosen": 82.93448638916016,
+      "logits/rejected": 82.9182357788086,
+      "logps/chosen": -30.893056869506836,
+      "logps/rejected": -31.30733299255371,
+      "loss": 1.6619,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.2736847400665283,
+      "rewards/margins": 0.40879687666893005,
+      "rewards/rejected": -0.13511209189891815,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 8768.0,
+      "grad_norm": 130.0,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 81.00895690917969,
-      "logits/rejected": 80.99317169189453,
-      "logps/chosen": -30.64133644104004,
-      "logps/rejected": -31.213275909423828,
-      "loss": 2421.4789,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.001250783447176218,
-      "rewards/margins": 0.008703308179974556,
-      "rewards/rejected": -0.007452525198459625,
+      "logits/chosen": 84.62263488769531,
+      "logits/rejected": 84.59950256347656,
+      "logps/chosen": -30.79315757751465,
+      "logps/rejected": -30.666366577148438,
+      "loss": 2.2923,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.02406691387295723,
+      "rewards/margins": 0.15444278717041016,
+      "rewards/rejected": -0.17850971221923828,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 13248.0,
+      "grad_norm": 536.0,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 76.1690673828125,
-      "logits/rejected": 76.11750793457031,
-      "logps/chosen": -33.94331741333008,
-      "logps/rejected": -33.27939224243164,
-      "loss": 2352.7865,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.008094213902950287,
-      "rewards/margins": 0.01595410332083702,
-      "rewards/rejected": -0.007859889417886734,
+      "logits/chosen": 79.89015197753906,
+      "logits/rejected": 79.83697509765625,
+      "logps/chosen": -34.30753707885742,
+      "logps/rejected": -32.36396026611328,
+      "loss": 3.1221,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.40067988634109497,
+      "rewards/margins": 0.2841777205467224,
+      "rewards/rejected": 0.11650214344263077,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.06681823730469,
-      "eval_logits/rejected": 98.04421997070312,
-      "eval_logps/chosen": -32.92998123168945,
-      "eval_logps/rejected": -36.8482551574707,
-      "eval_loss": 2463.577880859375,
-      "eval_rewards/accuracies": 0.5776578187942505,
-      "eval_rewards/chosen": -0.004868050571531057,
-      "eval_rewards/margins": 0.003949206322431564,
-      "eval_rewards/rejected": -0.008817256428301334,
-      "eval_runtime": 104.1998,
-      "eval_samples_per_second": 3.292,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 99.16912841796875,
+      "eval_logits/rejected": 99.15798950195312,
+      "eval_logps/chosen": -32.52165603637695,
+      "eval_logps/rejected": -36.10483932495117,
+      "eval_loss": 1.716450572013855,
+      "eval_rewards/accuracies": 0.5278239250183105,
+      "eval_rewards/chosen": -0.0706300213932991,
+      "eval_rewards/margins": 0.05385042726993561,
+      "eval_rewards/rejected": -0.12448045611381531,
+      "eval_runtime": 103.9441,
+      "eval_samples_per_second": 3.3,
+      "eval_steps_per_second": 0.414,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 17664.0,
+      "grad_norm": 392.0,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 78.7789535522461,
-      "logits/rejected": 78.68719482421875,
-      "logps/chosen": -33.58464431762695,
-      "logps/rejected": -36.030696868896484,
-      "loss": 2371.9807,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0036107641644775867,
-      "rewards/margins": 0.01405680924654007,
-      "rewards/rejected": -0.010446044616401196,
+      "logits/chosen": 82.42179870605469,
+      "logits/rejected": 82.31734466552734,
+      "logps/chosen": -33.38166427612305,
+      "logps/rejected": -35.02565383911133,
+      "loss": 2.7032,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.507649302482605,
+      "rewards/margins": 0.5432528257369995,
+      "rewards/rejected": -0.03560344874858856,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 13888.0,
+      "grad_norm": 474.0,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 80.73214721679688,
-      "logits/rejected": 80.83171081542969,
-      "logps/chosen": -31.50435447692871,
-      "logps/rejected": -31.954601287841797,
-      "loss": 2344.5365,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.004446002654731274,
-      "rewards/margins": 0.016702493652701378,
-      "rewards/rejected": -0.012256490997970104,
+      "logits/chosen": 84.43553161621094,
+      "logits/rejected": 84.52201080322266,
+      "logps/chosen": -31.842187881469727,
+      "logps/rejected": -30.83688735961914,
+      "loss": 3.6089,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09608902782201767,
+      "rewards/margins": 0.1932254582643509,
+      "rewards/rejected": -0.09713643789291382,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 13504.0,
+      "grad_norm": 324.0,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 77.71788024902344,
-      "logits/rejected": 77.77435302734375,
-      "logps/chosen": -32.75175857543945,
-      "logps/rejected": -35.30657196044922,
-      "loss": 2376.5023,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0005398989887908101,
-      "rewards/margins": 0.013484098017215729,
-      "rewards/rejected": -0.01294419914484024,
+      "logits/chosen": 81.6541976928711,
+      "logits/rejected": 81.72249603271484,
+      "logps/chosen": -32.656639099121094,
+      "logps/rejected": -33.97739791870117,
+      "loss": 1.7997,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.1341962367296219,
+      "rewards/margins": 0.10292205959558487,
+      "rewards/rejected": 0.03127415105700493,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 15168.0,
+      "grad_norm": 328.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 79.85724639892578,
-      "logits/rejected": 80.16901397705078,
-      "logps/chosen": -31.281352996826172,
-      "logps/rejected": -32.81675338745117,
-      "loss": 2341.0422,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.004067116882652044,
-      "rewards/margins": 0.016914214938879013,
-      "rewards/rejected": -0.012847095727920532,
+      "logits/chosen": 83.9974365234375,
+      "logits/rejected": 84.2688217163086,
+      "logps/chosen": -31.19537353515625,
+      "logps/rejected": -31.57509994506836,
+      "loss": 2.2222,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.443420946598053,
+      "rewards/margins": 0.4821735918521881,
+      "rewards/rejected": -0.038752567023038864,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 14208.0,
+      "grad_norm": 404.0,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 78.39628601074219,
-      "logits/rejected": 78.45399475097656,
-      "logps/chosen": -27.750865936279297,
-      "logps/rejected": -31.25889015197754,
-      "loss": 2391.2072,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.0015497003914788365,
-      "rewards/margins": 0.012031818740069866,
-      "rewards/rejected": -0.013581519015133381,
+      "logits/chosen": 82.76817321777344,
+      "logits/rejected": 82.85389709472656,
+      "logps/chosen": -27.292465209960938,
+      "logps/rejected": -29.717370986938477,
+      "loss": 2.0726,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.27308744192123413,
+      "rewards/margins": 0.10805711895227432,
+      "rewards/rejected": 0.1650303304195404,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 14656.0,
+      "grad_norm": 672.0,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 75.4200668334961,
-      "logits/rejected": 75.56632995605469,
-      "logps/chosen": -31.133581161499023,
-      "logps/rejected": -38.1984977722168,
-      "loss": 2269.6586,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.004274115897715092,
-      "rewards/margins": 0.025242850184440613,
-      "rewards/rejected": -0.020968737080693245,
+      "logits/chosen": 79.98811340332031,
+      "logits/rejected": 80.15504455566406,
+      "logps/chosen": -31.048583984375,
+      "logps/rejected": -35.97705841064453,
+      "loss": 3.1081,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.4611719250679016,
+      "rewards/margins": 0.3490581512451172,
+      "rewards/rejected": 0.112113818526268,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 12224.0,
+      "grad_norm": 211.0,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 74.54602813720703,
-      "logits/rejected": 74.57357025146484,
-      "logps/chosen": -31.873310089111328,
-      "logps/rejected": -33.126651763916016,
-      "loss": 2359.4146,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.0009438649867661297,
-      "rewards/margins": 0.015268439427018166,
-      "rewards/rejected": -0.016212303191423416,
+      "logits/chosen": 79.3856201171875,
+      "logits/rejected": 79.41062927246094,
+      "logps/chosen": -31.38250732421875,
+      "logps/rejected": -31.722015380859375,
+      "loss": 2.0801,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3567788600921631,
+      "rewards/margins": 0.5517110228538513,
+      "rewards/rejected": -0.194932222366333,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 19328.0,
+      "grad_norm": 498.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 77.44233703613281,
-      "logits/rejected": 77.22230529785156,
-      "logps/chosen": -32.219966888427734,
-      "logps/rejected": -31.11635398864746,
-      "loss": 2409.1738,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.004100737161934376,
-      "rewards/margins": 0.010392850264906883,
-      "rewards/rejected": -0.014493587426841259,
+      "logits/chosen": 81.8171157836914,
+      "logits/rejected": 81.62063598632812,
+      "logps/chosen": -31.32455062866211,
+      "logps/rejected": -29.448989868164062,
+      "loss": 2.1832,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.43680915236473083,
+      "rewards/margins": 0.2406042516231537,
+      "rewards/rejected": 0.19620487093925476,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 12736.0,
+      "grad_norm": 262.0,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 77.4681625366211,
-      "logits/rejected": 77.38896942138672,
-      "logps/chosen": -34.16426086425781,
-      "logps/rejected": -34.382694244384766,
-      "loss": 2287.8654,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0006001373985782266,
-      "rewards/margins": 0.023058168590068817,
-      "rewards/rejected": -0.023658307269215584,
+      "logits/chosen": 82.0579833984375,
+      "logits/rejected": 81.96476745605469,
+      "logps/chosen": -33.8171272277832,
+      "logps/rejected": -31.962472915649414,
+      "loss": 3.4215,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.25841274857521057,
+      "rewards/margins": 0.2094600647687912,
+      "rewards/rejected": 0.04895265772938728,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 14720.0,
+      "grad_norm": 169.0,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 73.11376953125,
-      "logits/rejected": 73.2473373413086,
-      "logps/chosen": -33.0648193359375,
-      "logps/rejected": -30.62347984313965,
-      "loss": 2331.667,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0034088939428329468,
-      "rewards/margins": 0.01862344518303871,
-      "rewards/rejected": -0.01521455030888319,
+      "logits/chosen": 77.75779724121094,
+      "logits/rejected": 77.80491638183594,
+      "logps/chosen": -32.91708755493164,
+      "logps/rejected": -29.0297908782959,
+      "loss": 2.4404,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.4397595524787903,
+      "rewards/margins": 0.37474992871284485,
+      "rewards/rejected": 0.06500961631536484,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 97.80475616455078,
-      "eval_logits/rejected": 97.77122497558594,
-      "eval_logps/chosen": -33.280792236328125,
-      "eval_logps/rejected": -37.28933334350586,
-      "eval_loss": 2455.659423828125,
-      "eval_rewards/accuracies": 0.5772424936294556,
-      "eval_rewards/chosen": -0.008376174606382847,
-      "eval_rewards/margins": 0.004851866513490677,
-      "eval_rewards/rejected": -0.013228042051196098,
-      "eval_runtime": 104.0404,
-      "eval_samples_per_second": 3.297,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 99.06168365478516,
+      "eval_logits/rejected": 99.05675506591797,
+      "eval_logps/chosen": -32.30947494506836,
+      "eval_logps/rejected": -35.82986068725586,
+      "eval_loss": 1.5173850059509277,
+      "eval_rewards/accuracies": 0.510797381401062,
+      "eval_rewards/chosen": 0.12033051997423172,
+      "eval_rewards/margins": -0.0026713553816080093,
+      "eval_rewards/rejected": 0.12300187349319458,
+      "eval_runtime": 103.9278,
+      "eval_samples_per_second": 3.3,
+      "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 13696.0,
+      "grad_norm": 928.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 80.57582092285156,
-      "logits/rejected": 80.58354949951172,
-      "logps/chosen": -30.828479766845703,
-      "logps/rejected": -34.02311325073242,
-      "loss": 2330.1223,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.00035051433951593935,
-      "rewards/margins": 0.018584031611680984,
-      "rewards/rejected": -0.018233519047498703,
+      "logits/chosen": 84.6921157836914,
+      "logits/rejected": 84.73957824707031,
+      "logps/chosen": -30.482025146484375,
+      "logps/rejected": -32.063751220703125,
+      "loss": 2.7405,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.3433563709259033,
+      "rewards/margins": 0.22095146775245667,
+      "rewards/rejected": 0.12240489572286606,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 14656.0,
+      "grad_norm": 406.0,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 77.68934631347656,
-      "logits/rejected": 77.70649719238281,
-      "logps/chosen": -31.195690155029297,
-      "logps/rejected": -30.364582061767578,
-      "loss": 2329.8537,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.004028716124594212,
-      "rewards/margins": 0.01847960613667965,
-      "rewards/rejected": -0.014450890012085438,
+      "logits/chosen": 82.20662689208984,
+      "logits/rejected": 82.20085906982422,
+      "logps/chosen": -31.305639266967773,
+      "logps/rejected": -28.786062240600586,
+      "loss": 2.2147,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.2636275887489319,
+      "rewards/margins": 0.14354394376277924,
+      "rewards/rejected": 0.12008367478847504,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 16064.0,
+      "grad_norm": 366.0,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 74.70455932617188,
-      "logits/rejected": 74.74969482421875,
-      "logps/chosen": -29.661697387695312,
-      "logps/rejected": -34.58174133300781,
-      "loss": 2253.0625,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.006909091956913471,
-      "rewards/margins": 0.026377875357866287,
-      "rewards/rejected": -0.01946878433227539,
+      "logits/chosen": 79.59921264648438,
+      "logits/rejected": 79.63972473144531,
+      "logps/chosen": -29.793676376342773,
+      "logps/rejected": -32.53992462158203,
+      "loss": 2.2293,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.5030375719070435,
+      "rewards/margins": 0.41758814454078674,
+      "rewards/rejected": 0.08544941991567612,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 17024.0,
+      "grad_norm": 300.0,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 79.37252044677734,
-      "logits/rejected": 79.41529846191406,
-      "logps/chosen": -33.01287841796875,
-      "logps/rejected": -35.60404586791992,
-      "loss": 2279.2205,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.001992682693526149,
-      "rewards/margins": 0.024346230551600456,
-      "rewards/rejected": -0.02235354855656624,
+      "logits/chosen": 83.62577056884766,
+      "logits/rejected": 83.62586975097656,
+      "logps/chosen": -32.759605407714844,
+      "logps/rejected": -33.222259521484375,
+      "loss": 2.4595,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.40729111433029175,
+      "rewards/margins": 0.2755037844181061,
+      "rewards/rejected": 0.13178732991218567,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 13824.0,
+      "grad_norm": 302.0,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 78.25090789794922,
-      "logits/rejected": 78.26826477050781,
-      "logps/chosen": -33.182167053222656,
-      "logps/rejected": -34.755130767822266,
-      "loss": 2306.5689,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.0046157208271324635,
-      "rewards/margins": 0.02124902606010437,
-      "rewards/rejected": -0.016633303835988045,
+      "logits/chosen": 82.8216781616211,
+      "logits/rejected": 82.81562805175781,
+      "logps/chosen": -33.33207702636719,
+      "logps/rejected": -32.93452835083008,
+      "loss": 1.9575,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2804979979991913,
+      "rewards/margins": 0.13895203173160553,
+      "rewards/rejected": 0.14154598116874695,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 12672.0,
+      "grad_norm": 218.0,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 79.92387390136719,
-      "logits/rejected": 79.95523071289062,
-      "logps/chosen": -28.899118423461914,
-      "logps/rejected": -32.96432113647461,
-      "loss": 2318.801,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.006530913524329662,
-      "rewards/margins": 0.01967565529048443,
-      "rewards/rejected": -0.013144740834832191,
+      "logits/chosen": 84.19699096679688,
+      "logits/rejected": 84.23506164550781,
+      "logps/chosen": -28.99674415588379,
+      "logps/rejected": -31.437408447265625,
+      "loss": 2.09,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.4999212324619293,
+      "rewards/margins": 0.3087221086025238,
+      "rewards/rejected": 0.19119907915592194,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 16192.0,
+      "grad_norm": 508.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 79.36608123779297,
-      "logits/rejected": 79.37577819824219,
-      "logps/chosen": -33.16106033325195,
-      "logps/rejected": -36.981719970703125,
-      "loss": 2360.0264,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.0028489597607403994,
-      "rewards/margins": 0.01587451994419098,
-      "rewards/rejected": -0.01872348040342331,
+      "logits/chosen": 83.60005187988281,
+      "logits/rejected": 83.62088012695312,
+      "logps/chosen": -32.41377258300781,
+      "logps/rejected": -34.801570892333984,
+      "loss": 3.3754,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.41615089774131775,
+      "rewards/margins": 0.13912765681743622,
+      "rewards/rejected": 0.2770232558250427,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 12800.0,
+      "grad_norm": 194.0,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 72.95059967041016,
-      "logits/rejected": 72.81791687011719,
-      "logps/chosen": -30.571094512939453,
-      "logps/rejected": -29.748737335205078,
-      "loss": 2355.1461,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 5.033141860621981e-05,
-      "rewards/margins": 0.015931954607367516,
-      "rewards/rejected": -0.015881624072790146,
+      "logits/chosen": 77.62187194824219,
+      "logits/rejected": 77.48942565917969,
+      "logps/chosen": -30.352733612060547,
+      "logps/rejected": -28.094280242919922,
+      "loss": 2.3864,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.20105357468128204,
+      "rewards/margins": 0.14138910174369812,
+      "rewards/rejected": 0.059664465487003326,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 2389.4745231331167,
-      "train_runtime": 2559.5069,
+      "train_loss": 2.2499145250041765,
+      "train_runtime": 2558.7273,
       "train_samples_per_second": 1.203,
       "train_steps_per_second": 0.15
     }