Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-magpi-high-bleu-3-epochs
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0049
-- Rewards/chosen: -1.6090
-- Rewards/rejected: -45.3616
+- Loss: 0.0050
+- Rewards/chosen: -1.4582
+- Rewards/rejected: -44.8746
 - Rewards/accuracies: 0.9960
-- Rewards/margins: 43.7525
-- Logps/rejected: -5176.9458
-- Logps/chosen: -527.8860
-- Logits/rejected: -3.4269
-- Logits/chosen: -3.5551
+- Rewards/margins: 43.4164
+- Logps/rejected: -5128.2480
+- Logps/chosen: -512.8050
+- Logits/rejected: -3.4441
+- Logits/chosen: -3.5504
 
 ## Model description
 
@@ -62,12 +63,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0025        | 0.4739 | 50   | 0.0067          | -1.4593        | -36.8732         | 0.9940             | 35.4140         | -4328.1133     | -512.9101    | -3.0480         | -3.2146       |
-| 0.0035        | 0.9479 | 100  | 0.0053          | -1.9936        | -41.1186         | 0.9940             | 39.1250         | -4752.6543     | -566.3461    | -3.2382         | -3.4106       |
-| 0.0002        | 1.4218 | 150  | 0.0072          | -2.1152        | -46.0999         | 0.9940             | 43.9846         | -5250.7739     | -578.5039    | -3.5133         | -3.5737       |
-| 0.0           | 1.8957 | 200  | 0.0046          | -1.6298        | -44.8878         | 0.9960             | 43.2581         | -5129.5718     | -529.9568    | -3.3932         | -3.4839       |
-| 0.0           | 2.3697 | 250  | 0.0049          | -1.6103        | -45.2487         | 0.9960             | 43.6384         | -5165.6558     | -528.0136    | -3.4300         | -3.5565       |
-| 0.0           | 2.8436 | 300  | 0.0049          | -1.6090        | -45.3616         | 0.9960             | 43.7525         | -5176.9458     | -527.8860    | -3.4269         | -3.5551       |
+| 0.0025        | 0.4739 | 50   | 0.0067          | -1.4454        | -37.1575         | 0.9940             | 35.7120         | -4356.5356     | -511.5262    | -3.0434         | -3.2142       |
+| 0.0036        | 0.9479 | 100  | 0.0053          | -2.0303        | -41.2450         | 0.9940             | 39.2146         | -4765.2842     | -570.0164    | -3.2429         | -3.4104       |
+| 0.0001        | 1.4218 | 150  | 0.0070          | -1.9459        | -45.2030         | 0.9940             | 43.2570         | -5161.0879     | -561.5757    | -3.5068         | -3.5867       |
+| 0.0           | 1.8957 | 200  | 0.0047          | -1.4539        | -44.2686         | 0.9960             | 42.8147         | -5067.6450     | -512.3704    | -3.4229         | -3.5020       |
+| 0.0           | 2.3697 | 250  | 0.0050          | -1.4525        | -44.7537         | 0.9960             | 43.3012         | -5116.1577     | -512.2269    | -3.4445         | -3.5510       |
+| 0.0           | 2.8436 | 300  | 0.0050          | -1.4582        | -44.8746         | 0.9960             | 43.4164         | -5128.2480     | -512.8050    | -3.4441         | -3.5504       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 2.985781990521327,
+    "eval_logits/chosen": -3.554823398590088,
+    "eval_logits/rejected": -3.4271774291992188,
+    "eval_logps/chosen": -527.9577026367188,
+    "eval_logps/rejected": -5177.13427734375,
+    "eval_loss": 0.004887364339083433,
+    "eval_rewards/accuracies": 0.9959677457809448,
+    "eval_rewards/chosen": -1.6097602844238281,
+    "eval_rewards/margins": 43.75369644165039,
+    "eval_rewards/rejected": -45.363460540771484,
+    "eval_runtime": 195.0759,
+    "eval_samples": 3905,
+    "eval_samples_per_second": 20.018,
+    "eval_steps_per_second": 0.318,
     "total_flos": 0.0,
-    "train_loss": 0.04085985715484842,
-    "train_runtime": 9625.0686,
+    "train_loss": 0.04083177362173292,
+    "train_runtime": 9033.5209,
     "train_samples": 13500,
-    "train_samples_per_second": 4.208,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 4.483,
+    "train_steps_per_second": 0.035
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 2.985781990521327,
+    "eval_logits/chosen": -3.554823398590088,
+    "eval_logits/rejected": -3.4271774291992188,
+    "eval_logps/chosen": -527.9577026367188,
+    "eval_logps/rejected": -5177.13427734375,
+    "eval_loss": 0.004887364339083433,
+    "eval_rewards/accuracies": 0.9959677457809448,
+    "eval_rewards/chosen": -1.6097602844238281,
+    "eval_rewards/margins": 43.75369644165039,
+    "eval_rewards/rejected": -45.363460540771484,
+    "eval_runtime": 195.0759,
+    "eval_samples": 3905,
+    "eval_samples_per_second": 20.018,
+    "eval_steps_per_second": 0.318
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d251c4ef2b4a1a2fbdef1160fc39c3103e5d5886d52070b7b539e89cb8da01f1
+oid sha256:68975e1291478d1339fd59772dd3bda25df676f16a75793190e1e237d5b07cbf
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9218cc23d068a585d47810af43283eaf22c7531ec408d0a3f18ed1ac8fe9c327
+oid sha256:cf9410e725afbeb11aa75a576814b569404c46920bef499dccdea40e07762d23
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f84601a6e02f549d17a9f429968fd5d2f9d509ab2440df0d7286b977c477ba52
+oid sha256:d296637a91238fa45d093d1f90ca28ac5588378f56065bfa495eba3c173a2e75
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.985781990521327,
     "total_flos": 0.0,
-    "train_loss": 0.04085985715484842,
-    "train_runtime": 9625.0686,
+    "train_loss": 0.04083177362173292,
+    "train_runtime": 9033.5209,
     "train_samples": 13500,
-    "train_samples_per_second": 4.208,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 4.483,
+    "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,573 +10,573 @@
   "log_history": [
     {
       "epoch": 0.0947867298578199,
-      "grad_norm": 56.808507663630365,
+      "grad_norm": 56.951628924108704,
       "learning_rate": 1.5624999999999999e-07,
-      "logits/chosen": -2.8022360801696777,
-      "logits/rejected": -2.6993465423583984,
-      "logps/chosen": -354.06646728515625,
-      "logps/rejected": -648.7581787109375,
-      "loss": 0.6845,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.003776966128498316,
-      "rewards/margins": 0.015949796885252,
-      "rewards/rejected": -0.012172830291092396,
+      "logits/chosen": -2.8022689819335938,
+      "logits/rejected": -2.699367046356201,
+      "logps/chosen": -354.14007568359375,
+      "logps/rejected": -648.7852783203125,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0030409712344408035,
+      "rewards/margins": 0.015484926290810108,
+      "rewards/rejected": -0.01244395412504673,
       "step": 10
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 16.92917194646474,
+      "grad_norm": 16.911922497415656,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.845165729522705,
-      "logits/rejected": -2.729945421218872,
-      "logps/chosen": -361.77703857421875,
-      "logps/rejected": -731.990234375,
-      "loss": 0.449,
+      "logits/chosen": -2.8449482917785645,
+      "logits/rejected": -2.7297720909118652,
+      "logps/chosen": -361.7726135253906,
+      "logps/rejected": -731.9713134765625,
+      "loss": 0.4488,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.11443690210580826,
-      "rewards/margins": 0.7462291717529297,
-      "rewards/rejected": -0.6317921876907349,
+      "rewards/chosen": 0.11448182910680771,
+      "rewards/margins": 0.7460837364196777,
+      "rewards/rejected": -0.6316019892692566,
       "step": 20
     },
     {
       "epoch": 0.2843601895734597,
-      "grad_norm": 2.8859667385666943,
+      "grad_norm": 2.8879981399804886,
       "learning_rate": 4.6874999999999996e-07,
-      "logits/chosen": -2.9244141578674316,
-      "logits/rejected": -2.7600600719451904,
-      "logps/chosen": -344.0387268066406,
-      "logps/rejected": -1062.6480712890625,
-      "loss": 0.1129,
+      "logits/chosen": -2.924880027770996,
+      "logits/rejected": -2.7608063220977783,
+      "logps/chosen": -344.0640869140625,
+      "logps/rejected": -1062.529541015625,
+      "loss": 0.1128,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.4156733453273773,
-      "rewards/margins": 4.948118686676025,
-      "rewards/rejected": -4.532444953918457,
+      "rewards/chosen": 0.4154191017150879,
+      "rewards/margins": 4.946678638458252,
+      "rewards/rejected": -4.531259536743164,
       "step": 30
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 0.7962774018104511,
+      "grad_norm": 0.7967945507055681,
       "learning_rate": 4.990147841143461e-07,
-      "logits/chosen": -2.992558002471924,
-      "logits/rejected": -2.855602264404297,
-      "logps/chosen": -369.517822265625,
-      "logps/rejected": -2512.892822265625,
+      "logits/chosen": -2.9928297996520996,
+      "logits/rejected": -2.858860969543457,
+      "logps/chosen": -369.7523193359375,
+      "logps/rejected": -2523.788818359375,
       "loss": 0.0145,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.17123910784721375,
-      "rewards/margins": 18.776042938232422,
-      "rewards/rejected": -18.947282791137695,
+      "rewards/chosen": -0.17358417809009552,
+      "rewards/margins": 18.882659912109375,
+      "rewards/rejected": -19.056243896484375,
       "step": 40
     },
     {
       "epoch": 0.47393364928909953,
-      "grad_norm": 0.08077589023709618,
+      "grad_norm": 0.08242657747458541,
       "learning_rate": 4.950256493879794e-07,
-      "logits/chosen": -3.1441614627838135,
-      "logits/rejected": -3.0702505111694336,
-      "logps/chosen": -446.707763671875,
-      "logps/rejected": -3806.796875,
+      "logits/chosen": -3.1458115577697754,
+      "logits/rejected": -3.068504810333252,
+      "logps/chosen": -445.88641357421875,
+      "logps/rejected": -3839.385498046875,
       "loss": 0.0025,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.8762685656547546,
-      "rewards/margins": 31.019947052001953,
-      "rewards/rejected": -31.896215438842773,
+      "rewards/chosen": -0.8680551648139954,
+      "rewards/margins": 31.354045867919922,
+      "rewards/rejected": -32.22209930419922,
       "step": 50
     },
     {
       "epoch": 0.47393364928909953,
-      "eval_logits/chosen": -3.2145538330078125,
-      "eval_logits/rejected": -3.048049211502075,
-      "eval_logps/chosen": -512.9100952148438,
-      "eval_logps/rejected": -4328.11328125,
-      "eval_loss": 0.006741416174918413,
+      "eval_logits/chosen": -3.214230537414551,
+      "eval_logits/rejected": -3.0434162616729736,
+      "eval_logps/chosen": -511.5262451171875,
+      "eval_logps/rejected": -4356.53564453125,
+      "eval_loss": 0.006651720497757196,
       "eval_rewards/accuracies": 0.9939516186714172,
-      "eval_rewards/chosen": -1.4592840671539307,
-      "eval_rewards/margins": 35.413963317871094,
-      "eval_rewards/rejected": -36.87324523925781,
-      "eval_runtime": 193.9438,
-      "eval_samples_per_second": 20.135,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -1.4454454183578491,
+      "eval_rewards/margins": 35.71202850341797,
+      "eval_rewards/rejected": -37.157470703125,
+      "eval_runtime": 194.5294,
+      "eval_samples_per_second": 20.074,
+      "eval_steps_per_second": 0.319,
       "step": 50
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 0.08718232747940217,
+      "grad_norm": 0.08904936739654302,
       "learning_rate": 4.88020090697132e-07,
-      "logits/chosen": -3.2784461975097656,
-      "logits/rejected": -3.1442923545837402,
-      "logps/chosen": -567.5687255859375,
-      "logps/rejected": -4666.0283203125,
-      "loss": 0.0042,
+      "logits/chosen": -3.2791202068328857,
+      "logits/rejected": -3.141754150390625,
+      "logps/chosen": -564.9468383789062,
+      "logps/rejected": -4684.3271484375,
+      "loss": 0.004,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.9609228372573853,
-      "rewards/margins": 38.35578536987305,
-      "rewards/rejected": -40.31671142578125,
+      "rewards/chosen": -1.9347045421600342,
+      "rewards/margins": 38.56499099731445,
+      "rewards/rejected": -40.49969482421875,
       "step": 60
     },
     {
       "epoch": 0.6635071090047393,
-      "grad_norm": 1.5224160707137697,
+      "grad_norm": 1.511268095124282,
       "learning_rate": 4.780843509929904e-07,
-      "logits/chosen": -3.293713331222534,
-      "logits/rejected": -3.0942394733428955,
-      "logps/chosen": -608.1524047851562,
-      "logps/rejected": -4869.34912109375,
+      "logits/chosen": -3.2914862632751465,
+      "logits/rejected": -3.0883309841156006,
+      "logps/chosen": -603.4210205078125,
+      "logps/rejected": -4877.28662109375,
       "loss": 0.0021,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.6836161613464355,
-      "rewards/margins": 39.84333419799805,
-      "rewards/rejected": -42.52695083618164,
+      "rewards/chosen": -2.6363024711608887,
+      "rewards/margins": 39.97002410888672,
+      "rewards/rejected": -42.606327056884766,
       "step": 70
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 0.19554010480864858,
+      "grad_norm": 0.22202350824430725,
       "learning_rate": 4.6534074564712217e-07,
-      "logits/chosen": -3.4218146800994873,
-      "logits/rejected": -3.29943585395813,
-      "logps/chosen": -602.691650390625,
-      "logps/rejected": -5427.1826171875,
-      "loss": 0.0004,
+      "logits/chosen": -3.417383909225464,
+      "logits/rejected": -3.290362596511841,
+      "logps/chosen": -600.4118041992188,
+      "logps/rejected": -5436.11376953125,
+      "loss": 0.0005,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.5821824073791504,
-      "rewards/margins": 45.447879791259766,
-      "rewards/rejected": -48.030067443847656,
+      "rewards/chosen": -2.5593833923339844,
+      "rewards/margins": 45.55999755859375,
+      "rewards/rejected": -48.11937713623047,
       "step": 80
     },
     {
       "epoch": 0.8530805687203792,
-      "grad_norm": 2.1541635120991005,
+      "grad_norm": 2.0861019684034874,
       "learning_rate": 4.4994615667026846e-07,
-      "logits/chosen": -3.4852542877197266,
-      "logits/rejected": -3.398355484008789,
-      "logps/chosen": -631.3218994140625,
-      "logps/rejected": -5290.6767578125,
+      "logits/chosen": -3.4805240631103516,
+      "logits/rejected": -3.3906772136688232,
+      "logps/chosen": -624.0176391601562,
+      "logps/rejected": -5296.82275390625,
       "loss": 0.0018,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.7490692138671875,
-      "rewards/margins": 43.93152618408203,
-      "rewards/rejected": -46.68059539794922,
+      "rewards/chosen": -2.676025867462158,
+      "rewards/margins": 44.0660285949707,
+      "rewards/rejected": -46.7420539855957,
       "step": 90
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 2.8671974784822956,
+      "grad_norm": 2.8965011668216905,
       "learning_rate": 4.320901013934887e-07,
-      "logits/chosen": -3.422508716583252,
-      "logits/rejected": -3.361572742462158,
-      "logps/chosen": -554.526123046875,
-      "logps/rejected": -4800.9462890625,
-      "loss": 0.0035,
+      "logits/chosen": -3.4210407733917236,
+      "logits/rejected": -3.3643829822540283,
+      "logps/chosen": -556.0076904296875,
+      "logps/rejected": -4813.1806640625,
+      "loss": 0.0036,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.072134494781494,
-      "rewards/margins": 39.7642936706543,
-      "rewards/rejected": -41.836429595947266,
+      "rewards/chosen": -2.0869507789611816,
+      "rewards/margins": 39.87181854248047,
+      "rewards/rejected": -41.95877456665039,
       "step": 100
     },
     {
       "epoch": 0.9478672985781991,
-      "eval_logits/chosen": -3.410635232925415,
-      "eval_logits/rejected": -3.2381887435913086,
-      "eval_logps/chosen": -566.3461303710938,
-      "eval_logps/rejected": -4752.654296875,
-      "eval_loss": 0.005255497060716152,
+      "eval_logits/chosen": -3.4104061126708984,
+      "eval_logits/rejected": -3.2429261207580566,
+      "eval_logps/chosen": -570.0164184570312,
+      "eval_logps/rejected": -4765.2841796875,
+      "eval_loss": 0.0052900416776537895,
       "eval_rewards/accuracies": 0.9939516186714172,
-      "eval_rewards/chosen": -1.9936442375183105,
-      "eval_rewards/margins": 39.12500762939453,
-      "eval_rewards/rejected": -41.118648529052734,
-      "eval_runtime": 193.1039,
-      "eval_samples_per_second": 20.222,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -2.0303473472595215,
+      "eval_rewards/margins": 39.21460723876953,
+      "eval_rewards/rejected": -41.24495315551758,
+      "eval_runtime": 192.2337,
+      "eval_samples_per_second": 20.314,
+      "eval_steps_per_second": 0.323,
       "step": 100
     },
     {
       "epoch": 1.042654028436019,
-      "grad_norm": 1.2561873368154433,
+      "grad_norm": 1.2489542878599509,
       "learning_rate": 4.119923993874379e-07,
-      "logits/chosen": -3.4638073444366455,
-      "logits/rejected": -3.4071457386016846,
-      "logps/chosen": -544.2169189453125,
-      "logps/rejected": -5126.4560546875,
+      "logits/chosen": -3.4639148712158203,
+      "logits/rejected": -3.4126315116882324,
+      "logps/chosen": -549.92138671875,
+      "logps/rejected": -5150.29638671875,
       "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.89871084690094,
-      "rewards/margins": 42.90679168701172,
-      "rewards/rejected": -44.805503845214844,
+      "rewards/chosen": -1.9557552337646484,
+      "rewards/margins": 43.08815002441406,
+      "rewards/rejected": -45.04390335083008,
       "step": 110
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 0.9786979825165962,
+      "grad_norm": 0.919711694376481,
       "learning_rate": 3.899004663415083e-07,
-      "logits/chosen": -3.452301502227783,
-      "logits/rejected": -3.3307366371154785,
-      "logps/chosen": -527.18212890625,
-      "logps/rejected": -5159.8701171875,
+      "logits/chosen": -3.455725908279419,
+      "logits/rejected": -3.3397490978240967,
+      "logps/chosen": -534.6444702148438,
+      "logps/rejected": -5193.822265625,
       "loss": 0.0005,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.735828161239624,
-      "rewards/margins": 43.46116256713867,
-      "rewards/rejected": -45.196990966796875,
+      "rewards/chosen": -1.8104517459869385,
+      "rewards/margins": 43.72606658935547,
+      "rewards/rejected": -45.53651809692383,
       "step": 120
     },
     {
       "epoch": 1.2322274881516588,
-      "grad_norm": 0.04433484422705367,
+      "grad_norm": 0.03772744312797018,
       "learning_rate": 3.6608626821692824e-07,
-      "logits/chosen": -3.4930477142333984,
-      "logits/rejected": -3.4833171367645264,
-      "logps/chosen": -507.93817138671875,
-      "logps/rejected": -5810.45654296875,
+      "logits/chosen": -3.503054141998291,
+      "logits/rejected": -3.4913394451141357,
+      "logps/chosen": -509.2953186035156,
+      "logps/rejected": -5831.84228515625,
       "loss": 0.0012,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.616611123085022,
-      "rewards/margins": 49.649314880371094,
-      "rewards/rejected": -51.26592254638672,
+      "rewards/chosen": -1.6301825046539307,
+      "rewards/margins": 49.84960174560547,
+      "rewards/rejected": -51.47977828979492,
       "step": 130
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 0.0001033743847851317,
+      "grad_norm": 0.00011722006953608906,
       "learning_rate": 3.408429731701635e-07,
-      "logits/chosen": -3.602503538131714,
-      "logits/rejected": -3.6203441619873047,
-      "logps/chosen": -699.213623046875,
-      "logps/rejected": -5553.19384765625,
+      "logits/chosen": -3.636444091796875,
+      "logits/rejected": -3.614245891571045,
+      "logps/chosen": -664.00341796875,
+      "logps/rejected": -5503.0537109375,
       "loss": 0.0001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -3.502641201019287,
-      "rewards/margins": 45.56864547729492,
-      "rewards/rejected": -49.07128143310547,
+      "rewards/chosen": -3.150538682937622,
+      "rewards/margins": 45.41934585571289,
+      "rewards/rejected": -48.56988525390625,
       "step": 140
     },
     {
       "epoch": 1.4218009478672986,
-      "grad_norm": 0.001084875953335538,
+      "grad_norm": 0.0013414969188062405,
       "learning_rate": 3.144813424636031e-07,
-      "logits/chosen": -3.7745907306671143,
-      "logits/rejected": -3.7177462577819824,
-      "logps/chosen": -846.7849731445312,
-      "logps/rejected": -5827.8447265625,
-      "loss": 0.0002,
+      "logits/chosen": -3.788306713104248,
+      "logits/rejected": -3.686079740524292,
+      "logps/chosen": -791.1682739257812,
+      "logps/rejected": -5721.5634765625,
+      "loss": 0.0001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.706751823425293,
-      "rewards/margins": 47.121307373046875,
-      "rewards/rejected": -51.82805633544922,
+      "rewards/chosen": -4.1505842208862305,
+      "rewards/margins": 46.614662170410156,
+      "rewards/rejected": -50.7652473449707,
       "step": 150
     },
     {
       "epoch": 1.4218009478672986,
-      "eval_logits/chosen": -3.573683977127075,
-      "eval_logits/rejected": -3.5132687091827393,
-      "eval_logps/chosen": -578.50390625,
-      "eval_logps/rejected": -5250.77392578125,
-      "eval_loss": 0.007219326216727495,
+      "eval_logits/chosen": -3.5867350101470947,
+      "eval_logits/rejected": -3.5067942142486572,
+      "eval_logps/chosen": -561.57568359375,
+      "eval_logps/rejected": -5161.087890625,
+      "eval_loss": 0.006992733106017113,
       "eval_rewards/accuracies": 0.9939516186714172,
-      "eval_rewards/chosen": -2.1152215003967285,
-      "eval_rewards/margins": 43.9846305847168,
-      "eval_rewards/rejected": -46.099853515625,
-      "eval_runtime": 192.0822,
-      "eval_samples_per_second": 20.33,
+      "eval_rewards/chosen": -1.9459394216537476,
+      "eval_rewards/margins": 43.25704574584961,
+      "eval_rewards/rejected": -45.2029914855957,
+      "eval_runtime": 191.7726,
+      "eval_samples_per_second": 20.363,
       "eval_steps_per_second": 0.323,
       "step": 150
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 0.00020050840378569657,
+      "grad_norm": 0.0004138099071654368,
       "learning_rate": 2.8732590479375165e-07,
-      "logits/chosen": -3.562108278274536,
-      "logits/rejected": -3.6045074462890625,
-      "logps/chosen": -542.7542724609375,
-      "logps/rejected": -5262.7451171875,
+      "logits/chosen": -3.556847333908081,
+      "logits/rejected": -3.5835862159729004,
+      "logps/chosen": -528.8604736328125,
+      "logps/rejected": -5157.8740234375,
       "loss": 0.0003,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.8957535028457642,
-      "rewards/margins": 44.86753463745117,
-      "rewards/rejected": -46.76329040527344,
+      "rewards/chosen": -1.7568155527114868,
+      "rewards/margins": 43.957759857177734,
+      "rewards/rejected": -45.714576721191406,
       "step": 160
     },
     {
       "epoch": 1.6113744075829384,
-      "grad_norm": 0.0014969345497476303,
+      "grad_norm": 0.0016286137021698196,
       "learning_rate": 2.597109611334169e-07,
-      "logits/chosen": -3.5832862854003906,
-      "logits/rejected": -3.668252944946289,
-      "logps/chosen": -530.57373046875,
-      "logps/rejected": -5514.72509765625,
-      "loss": 0.0002,
+      "logits/chosen": -3.579390287399292,
+      "logits/rejected": -3.6478075981140137,
+      "logps/chosen": -520.5675048828125,
+      "logps/rejected": -5432.5673828125,
+      "loss": 0.0001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.7220637798309326,
-      "rewards/margins": 47.27531814575195,
-      "rewards/rejected": -48.99738311767578,
+      "rewards/chosen": -1.6220014095306396,
+      "rewards/margins": 46.55379867553711,
+      "rewards/rejected": -48.17579650878906,
       "step": 170
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 0.012284347539022291,
+      "grad_norm": 0.00799320909391895,
       "learning_rate": 2.3197646927086694e-07,
-      "logits/chosen": -3.544964551925659,
-      "logits/rejected": -3.628335952758789,
-      "logps/chosen": -555.7106323242188,
-      "logps/rejected": -5537.78515625,
-      "loss": 0.0082,
+      "logits/chosen": -3.5350117683410645,
+      "logits/rejected": -3.6110050678253174,
+      "logps/chosen": -534.5997314453125,
+      "logps/rejected": -5420.73583984375,
+      "loss": 0.0075,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.8591806888580322,
-      "rewards/margins": 47.509742736816406,
-      "rewards/rejected": -49.36892318725586,
+      "rewards/chosen": -1.6480720043182373,
+      "rewards/margins": 46.55036163330078,
+      "rewards/rejected": -48.19843292236328,
       "step": 180
     },
     {
       "epoch": 1.8009478672985781,
-      "grad_norm": 0.0019233280418011931,
+      "grad_norm": 0.0014081828819370304,
       "learning_rate": 2.0446385870993467e-07,
-      "logits/chosen": -3.525050640106201,
-      "logits/rejected": -3.5435047149658203,
-      "logps/chosen": -543.8511352539062,
-      "logps/rejected": -5155.20263671875,
+      "logits/chosen": -3.5267558097839355,
+      "logits/rejected": -3.5355076789855957,
+      "logps/chosen": -524.6720581054688,
+      "logps/rejected": -5069.0888671875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6305545568466187,
-      "rewards/margins": 43.96278381347656,
-      "rewards/rejected": -45.59334182739258,
+      "rewards/chosen": -1.4387648105621338,
+      "rewards/margins": 43.29344177246094,
+      "rewards/rejected": -44.73220443725586,
       "step": 190
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 0.022433128971849518,
+      "grad_norm": 0.018000801767423476,
       "learning_rate": 1.775118274523545e-07,
-      "logits/chosen": -3.5229125022888184,
-      "logits/rejected": -3.511779308319092,
-      "logps/chosen": -507.39569091796875,
-      "logps/rejected": -5092.2197265625,
+      "logits/chosen": -3.5183377265930176,
+      "logits/rejected": -3.5119102001190186,
+      "logps/chosen": -486.629150390625,
+      "logps/rejected": -5021.52490234375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.549849510192871,
-      "rewards/margins": 43.13224411010742,
-      "rewards/rejected": -44.682090759277344,
+      "rewards/chosen": -1.3421844244003296,
+      "rewards/margins": 42.632965087890625,
+      "rewards/rejected": -43.97514724731445,
       "step": 200
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": -3.4838595390319824,
-      "eval_logits/rejected": -3.393195152282715,
-      "eval_logps/chosen": -529.9568481445312,
-      "eval_logps/rejected": -5129.57177734375,
-      "eval_loss": 0.004649566486477852,
+      "eval_logits/chosen": -3.502014398574829,
+      "eval_logits/rejected": -3.422856092453003,
+      "eval_logps/chosen": -512.3704223632812,
+      "eval_logps/rejected": -5067.64501953125,
+      "eval_loss": 0.004733214620500803,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -1.6297515630722046,
-      "eval_rewards/margins": 43.258079528808594,
-      "eval_rewards/rejected": -44.88783264160156,
-      "eval_runtime": 193.4054,
-      "eval_samples_per_second": 20.191,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -1.4538869857788086,
+      "eval_rewards/margins": 42.814674377441406,
+      "eval_rewards/rejected": -44.26856231689453,
+      "eval_runtime": 194.1121,
+      "eval_samples_per_second": 20.117,
+      "eval_steps_per_second": 0.319,
       "step": 200
     },
     {
       "epoch": 1.9905213270142181,
-      "grad_norm": 1.2842516744764987,
+      "grad_norm": 0.9312964869423628,
       "learning_rate": 1.514521724066537e-07,
-      "logits/chosen": -3.542708158493042,
-      "logits/rejected": -3.5746352672576904,
-      "logps/chosen": -553.3961181640625,
-      "logps/rejected": -5135.65087890625,
+      "logits/chosen": -3.540240526199341,
+      "logits/rejected": -3.5632777214050293,
+      "logps/chosen": -531.4307861328125,
+      "logps/rejected": -5061.63818359375,
       "loss": 0.0002,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.7059533596038818,
-      "rewards/margins": 43.47618865966797,
-      "rewards/rejected": -45.18214797973633,
+      "rewards/chosen": -1.486299753189087,
+      "rewards/margins": 42.955726623535156,
+      "rewards/rejected": -44.44202423095703,
       "step": 210
     },
     {
       "epoch": 2.085308056872038,
-      "grad_norm": 0.02286366102698996,
+      "grad_norm": 0.029566978048640967,
       "learning_rate": 1.266057047539568e-07,
-      "logits/chosen": -3.5124752521514893,
-      "logits/rejected": -3.5435707569122314,
-      "logps/chosen": -497.9325256347656,
-      "logps/rejected": -5323.30126953125,
+      "logits/chosen": -3.5052971839904785,
+      "logits/rejected": -3.5332977771759033,
+      "logps/chosen": -477.3848571777344,
+      "logps/rejected": -5269.00390625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6141698360443115,
-      "rewards/margins": 45.36501693725586,
-      "rewards/rejected": -46.97918701171875,
+      "rewards/chosen": -1.4086942672729492,
+      "rewards/margins": 45.027523040771484,
+      "rewards/rejected": -46.43621826171875,
       "step": 220
     },
     {
       "epoch": 2.1800947867298577,
-      "grad_norm": 0.000533688233348108,
+      "grad_norm": 0.0005556188331340245,
       "learning_rate": 1.032783005551884e-07,
-      "logits/chosen": -3.5625851154327393,
-      "logits/rejected": -3.5691921710968018,
-      "logps/chosen": -492.0570373535156,
-      "logps/rejected": -4921.8291015625,
+      "logits/chosen": -3.5509438514709473,
+      "logits/rejected": -3.5611331462860107,
+      "logps/chosen": -473.364501953125,
+      "logps/rejected": -4865.369140625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.4992796182632446,
-      "rewards/margins": 41.6370735168457,
-      "rewards/rejected": -43.1363525390625,
+      "rewards/chosen": -1.312354326248169,
+      "rewards/margins": 41.259403228759766,
+      "rewards/rejected": -42.57175827026367,
       "step": 230
     },
     {
       "epoch": 2.2748815165876777,
-      "grad_norm": 0.006381253133428834,
+      "grad_norm": 0.005629678669869344,
       "learning_rate": 8.175713521924976e-08,
-      "logits/chosen": -3.5869510173797607,
-      "logits/rejected": -3.523674726486206,
-      "logps/chosen": -516.2166748046875,
-      "logps/rejected": -5140.20703125,
+      "logits/chosen": -3.5678086280822754,
+      "logits/rejected": -3.5121123790740967,
+      "logps/chosen": -496.83258056640625,
+      "logps/rejected": -5081.9599609375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6389557123184204,
-      "rewards/margins": 43.762550354003906,
-      "rewards/rejected": -45.40150833129883,
+      "rewards/chosen": -1.4451147317886353,
+      "rewards/margins": 43.37391662597656,
+      "rewards/rejected": -44.81903839111328,
       "step": 240
     },
     {
       "epoch": 2.3696682464454977,
-      "grad_norm": 0.000733038849174391,
+      "grad_norm": 0.0012113886351427462,
       "learning_rate": 6.230714818829733e-08,
-      "logits/chosen": -3.5448238849639893,
-      "logits/rejected": -3.516840696334839,
-      "logps/chosen": -503.037841796875,
-      "logps/rejected": -5467.2509765625,
+      "logits/chosen": -3.530911922454834,
+      "logits/rejected": -3.5102057456970215,
+      "logps/chosen": -484.5502014160156,
+      "logps/rejected": -5412.3271484375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.5125716924667358,
-      "rewards/margins": 47.35405731201172,
-      "rewards/rejected": -48.86663055419922,
+      "rewards/chosen": -1.327695608139038,
+      "rewards/margins": 46.98969268798828,
+      "rewards/rejected": -48.31739044189453,
       "step": 250
     },
     {
       "epoch": 2.3696682464454977,
-      "eval_logits/chosen": -3.556544065475464,
-      "eval_logits/rejected": -3.430008888244629,
-      "eval_logps/chosen": -528.0136108398438,
-      "eval_logps/rejected": -5165.65576171875,
-      "eval_loss": 0.004941246937960386,
+      "eval_logits/chosen": -3.5510308742523193,
+      "eval_logits/rejected": -3.444518566131592,
+      "eval_logps/chosen": -512.2269287109375,
+      "eval_logps/rejected": -5116.15771484375,
+      "eval_loss": 0.005008448380976915,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -1.6103183031082153,
-      "eval_rewards/margins": 43.63835144042969,
-      "eval_rewards/rejected": -45.24867248535156,
-      "eval_runtime": 191.8373,
-      "eval_samples_per_second": 20.356,
+      "eval_rewards/chosen": -1.4524519443511963,
+      "eval_rewards/margins": 43.301239013671875,
+      "eval_rewards/rejected": -44.753692626953125,
+      "eval_runtime": 192.1218,
+      "eval_samples_per_second": 20.326,
       "eval_steps_per_second": 0.323,
       "step": 250
     },
     {
       "epoch": 2.4644549763033177,
-      "grad_norm": 0.004934692810403007,
+      "grad_norm": 0.005272804838769864,
       "learning_rate": 4.516778136213037e-08,
-      "logits/chosen": -3.5642166137695312,
-      "logits/rejected": -3.541161298751831,
-      "logps/chosen": -490.9266662597656,
-      "logps/rejected": -5269.9482421875,
+      "logits/chosen": -3.5464816093444824,
+      "logits/rejected": -3.532754898071289,
+      "logps/chosen": -474.98077392578125,
+      "logps/rejected": -5214.1748046875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.523624300956726,
-      "rewards/margins": 45.255531311035156,
-      "rewards/rejected": -46.77915573120117,
+      "rewards/chosen": -1.3641650676727295,
+      "rewards/margins": 44.85725021362305,
+      "rewards/rejected": -46.22141647338867,
       "step": 260
     },
     {
       "epoch": 2.5592417061611377,
-      "grad_norm": 0.0159993223919006,
+      "grad_norm": 0.01760309981671165,
       "learning_rate": 3.055003141378948e-08,
-      "logits/chosen": -3.5474228858947754,
-      "logits/rejected": -3.5533015727996826,
-      "logps/chosen": -518.9056396484375,
-      "logps/rejected": -5904.6611328125,
+      "logits/chosen": -3.5305237770080566,
+      "logits/rejected": -3.543522357940674,
+      "logps/chosen": -502.1796875,
+      "logps/rejected": -5842.8251953125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.4330569505691528,
-      "rewards/margins": 51.321136474609375,
-      "rewards/rejected": -52.75419998168945,
+      "rewards/chosen": -1.2657973766326904,
+      "rewards/margins": 50.87003707885742,
+      "rewards/rejected": -52.135841369628906,
       "step": 270
     },
     {
       "epoch": 2.654028436018957,
-      "grad_norm": 0.008470858031121525,
+      "grad_norm": 0.01345213655983596,
       "learning_rate": 1.8633852284264508e-08,
-      "logits/chosen": -3.55708646774292,
-      "logits/rejected": -3.5458106994628906,
-      "logps/chosen": -536.460205078125,
-      "logps/rejected": -5575.57177734375,
+      "logits/chosen": -3.5437607765197754,
+      "logits/rejected": -3.537663221359253,
+      "logps/chosen": -519.03759765625,
+      "logps/rejected": -5507.5615234375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.506127119064331,
-      "rewards/margins": 47.6081657409668,
-      "rewards/rejected": -49.11429214477539,
+      "rewards/chosen": -1.3319001197814941,
+      "rewards/margins": 47.102291107177734,
+      "rewards/rejected": -48.4341926574707,
       "step": 280
     },
     {
       "epoch": 2.748815165876777,
-      "grad_norm": 5.3150365717798084e-05,
+      "grad_norm": 5.9654408780918595e-05,
       "learning_rate": 9.56593983327919e-09,
-      "logits/chosen": -3.5954432487487793,
-      "logits/rejected": -3.5634117126464844,
-      "logps/chosen": -543.16455078125,
-      "logps/rejected": -5410.59375,
+      "logits/chosen": -3.5722999572753906,
+      "logits/rejected": -3.5434532165527344,
+      "logps/chosen": -525.2794189453125,
+      "logps/rejected": -5359.7451171875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.641262412071228,
-      "rewards/margins": 46.54301071166992,
-      "rewards/rejected": -48.18427276611328,
+      "rewards/chosen": -1.4624111652374268,
+      "rewards/margins": 46.21337890625,
+      "rewards/rejected": -47.675785064697266,
       "step": 290
     },
     {
       "epoch": 2.843601895734597,
-      "grad_norm": 0.0011549345423847704,
+      "grad_norm": 0.0012624104591569302,
       "learning_rate": 3.4579259185321398e-09,
-      "logits/chosen": -3.5742130279541016,
-      "logits/rejected": -3.561877727508545,
-      "logps/chosen": -529.9727783203125,
-      "logps/rejected": -5293.2900390625,
+      "logits/chosen": -3.5550761222839355,
+      "logits/rejected": -3.541923999786377,
+      "logps/chosen": -513.0765380859375,
+      "logps/rejected": -5235.28759765625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.5660574436187744,
-      "rewards/margins": 45.59223175048828,
-      "rewards/rejected": -47.15829849243164,
+      "rewards/chosen": -1.3970950841903687,
+      "rewards/margins": 45.181175231933594,
+      "rewards/rejected": -46.578269958496094,
       "step": 300
     },
     {
       "epoch": 2.843601895734597,
-      "eval_logits/chosen": -3.555126428604126,
-      "eval_logits/rejected": -3.426910161972046,
-      "eval_logps/chosen": -527.8860473632812,
-      "eval_logps/rejected": -5176.94580078125,
-      "eval_loss": 0.004882320296019316,
+      "eval_logits/chosen": -3.5504369735717773,
+      "eval_logits/rejected": -3.444122552871704,
+      "eval_logps/chosen": -512.8049926757812,
+      "eval_logps/rejected": -5128.248046875,
+      "eval_loss": 0.004975645802915096,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -1.6090432405471802,
-      "eval_rewards/margins": 43.75252914428711,
-      "eval_rewards/rejected": -45.361572265625,
-      "eval_runtime": 192.0885,
-      "eval_samples_per_second": 20.329,
-      "eval_steps_per_second": 0.323,
+      "eval_rewards/chosen": -1.4582326412200928,
+      "eval_rewards/margins": 43.41635513305664,
+      "eval_rewards/rejected": -44.87459182739258,
+      "eval_runtime": 192.6295,
+      "eval_samples_per_second": 20.272,
+      "eval_steps_per_second": 0.322,
       "step": 300
     },
     {
       "epoch": 2.938388625592417,
-      "grad_norm": 0.0003810246486281626,
+      "grad_norm": 0.0005953504074610172,
       "learning_rate": 3.850041354441502e-10,
-      "logits/chosen": -3.588923931121826,
-      "logits/rejected": -3.5137507915496826,
-      "logps/chosen": -526.1129150390625,
-      "logps/rejected": -4852.4716796875,
+      "logits/chosen": -3.5716750621795654,
+      "logits/rejected": -3.5102698802948,
+      "logps/chosen": -509.0469665527344,
+      "logps/rejected": -4801.1611328125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6466729640960693,
-      "rewards/margins": 41.060447692871094,
-      "rewards/rejected": -42.70712661743164,
+      "rewards/chosen": -1.4760140180587769,
+      "rewards/margins": 40.71800994873047,
+      "rewards/rejected": -42.19402313232422,
       "step": 310
     },
     {
       "epoch": 2.985781990521327,
       "step": 315,
       "total_flos": 0.0,
-      "train_loss": 0.04085985715484842,
-      "train_runtime": 9625.0686,
-      "train_samples_per_second": 4.208,
-      "train_steps_per_second": 0.033
+      "train_loss": 0.04083177362173292,
+      "train_runtime": 9033.5209,
+      "train_samples_per_second": 4.483,
+      "train_steps_per_second": 0.035
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c09991d0dd4badd7eb3d51c710e71e2c2a5932e93fe69a025005d92a6da7e339
+oid sha256:ecf20cbd8067145a8eb8173b159aac02fda214ae80ecd675174fcf61768dabe8
 size 7544