Model save

README.md

@@ -2,15 +2,9 @@
 license: other
 base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- argilla/dpo-mix-7k
 model-index:
 - name: zephyr-7b-gemma-dpo
   results: []
@@ -21,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-gemma-dpo
 
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k dataset.
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4578
-- Rewards/chosen: -3.5474
-- Rewards/rejected: -5.2118
+- Loss: 0.4692
+- Rewards/chosen: -2.9810
+- Rewards/rejected: -4.6451
 - Rewards/accuracies: 0.7604
-- Rewards/margins: 1.6644
-- Logps/rejected: -466.3073
-- Logps/chosen: -434.6161
-- Logits/rejected: 90.1100
-- Logits/chosen: 96.0078
+- Rewards/margins: 1.6641
+- Logps/rejected: -453.0530
+- Logps/chosen: -422.7995
+- Logits/rejected: 91.0574
+- Logits/chosen: 96.7661
 
 ## Model description
 
@@ -68,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1582        | 1.8957 | 100  | 0.4427          | -3.5439        | -5.2417          | 0.7604             | 1.6978          | -466.9060      | -434.5467    | 90.1294         | 96.0169       |
+| 0.1591        | 1.8957 | 100  | 0.4692          | -2.9810        | -4.6451          | 0.7604             | 1.6641          | -453.0530      | -422.7995    | 91.0574         | 96.7661       |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,9 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 96.0078125,
-    "eval_logits/rejected": 90.1099853515625,
-    "eval_logps/chosen": -434.6161193847656,
-    "eval_logps/rejected": -466.3072814941406,
-    "eval_loss": 0.4578173756599426,
-    "eval_rewards/accuracies": 0.7604166865348816,
-    "eval_rewards/chosen": -3.5473945140838623,
-    "eval_rewards/margins": 1.6644223928451538,
-    "eval_rewards/rejected": -5.211816787719727,
-    "eval_runtime": 124.9704,
-    "eval_samples": 750,
-    "eval_samples_per_second": 6.001,
-    "eval_steps_per_second": 0.192,
     "total_flos": 0.0,
-    "train_loss": 0.391697224516135,
-    "train_runtime": 2305.723,
+    "train_loss": 0.3921648321243433,
+    "train_runtime": 1190.3032,
     "train_samples": 6750,
-    "train_samples_per_second": 5.855,
-    "train_steps_per_second": 0.045
+    "train_samples_per_second": 11.342,
+    "train_steps_per_second": 0.087
 }

config.json

@@ -24,6 +24,6 @@
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 256000
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57be32a3428e1f2d925d42c93da364a2ff0b1cec6450fae99b663844b4353911
+oid sha256:7c5a7396241c9049d5aa615a0081a16f922ac27a595fad308da7be35b297a132
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53b2b463e3960df5c2d6061c8298bfef88abf4cbb236fcf2ef79e7f8a296bb3d
+oid sha256:cd5dd371acf66dbd26a13036c2b42efee4bf540edd178d9cb3a225eccd87d21c
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e567f4df2dbadafbf8c26a32627496c7fbd6db12d83a3346773b6b9fac86a995
+oid sha256:836f574fd62d28f4d282b81d3712708535d3eadc45ddd0509c932611129327c5
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af13efaf67010f78192a6b8de5992c22ee84ebcdddcefd83cd8ad305d497ecc8
+oid sha256:85fb13399e2932fccc4212b3907e81ae00bcdf3dbbb785a234bceaa501d29091
 size 2113988336

runs/Apr27_13-04-12_gcp002/events.out.tfevents.1714225073.gcp002.2844214.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52fbc293ec4d235cf3059606bc8b85f87a5b0758b3840ab0f43bccf065f0dd9c
+size 13440

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.391697224516135,
-    "train_runtime": 2305.723,
+    "train_loss": 0.3921648321243433,
+    "train_runtime": 1190.3032,
     "train_samples": 6750,
-    "train_samples_per_second": 5.855,
-    "train_steps_per_second": 0.045
+    "train_samples_per_second": 11.342,
+    "train_steps_per_second": 0.087
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 132.14458710760462,
+      "grad_norm": 133.64647421295854,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": 117.53560638427734,
-      "logits/rejected": 126.8960952758789,
-      "logps/chosen": -335.40118408203125,
-      "logps/rejected": -439.16552734375,
+      "logits/chosen": 117.4909439086914,
+      "logits/rejected": 126.8502426147461,
+      "logps/chosen": -338.3250732421875,
+      "logps/rejected": -438.210205078125,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,178 +25,178 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 129.17456289198574,
+      "grad_norm": 137.17714765050428,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 135.00433349609375,
-      "logits/rejected": 138.3448028564453,
-      "logps/chosen": -395.86181640625,
-      "logps/rejected": -439.7121887207031,
-      "loss": 0.7118,
-      "rewards/accuracies": 0.4097222089767456,
-      "rewards/chosen": 0.006736809387803078,
-      "rewards/margins": 0.02597012370824814,
-      "rewards/rejected": -0.019233308732509613,
+      "logits/chosen": 135.0139923095703,
+      "logits/rejected": 138.361328125,
+      "logps/chosen": -397.126220703125,
+      "logps/rejected": -439.42083740234375,
+      "loss": 0.7143,
+      "rewards/accuracies": 0.3888888955116272,
+      "rewards/chosen": -0.02168009988963604,
+      "rewards/margins": -0.04445798695087433,
+      "rewards/rejected": 0.02277788519859314,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 126.56766201489141,
+      "grad_norm": 125.18497680199994,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": 121.39383697509766,
-      "logits/rejected": 125.12164306640625,
-      "logps/chosen": -371.44940185546875,
-      "logps/rejected": -424.017822265625,
-      "loss": 0.6352,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0481288805603981,
-      "rewards/margins": 0.24945171177387238,
-      "rewards/rejected": -0.20132283866405487,
+      "logits/chosen": 121.53276062011719,
+      "logits/rejected": 125.26307678222656,
+      "logps/chosen": -370.91107177734375,
+      "logps/rejected": -423.38677978515625,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08632902055978775,
+      "rewards/margins": 0.28587669134140015,
+      "rewards/rejected": -0.199547678232193,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 113.84442623510158,
+      "grad_norm": 115.98151525592598,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": 142.023681640625,
-      "logits/rejected": 135.55198669433594,
-      "logps/chosen": -417.58203125,
-      "logps/rejected": -462.6326599121094,
-      "loss": 0.5761,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.255840539932251,
-      "rewards/margins": 0.8756818771362305,
-      "rewards/rejected": -2.1315224170684814,
+      "logits/chosen": 142.67178344726562,
+      "logits/rejected": 136.16537475585938,
+      "logps/chosen": -415.01104736328125,
+      "logps/rejected": -460.3519592285156,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1033741235733032,
+      "rewards/margins": 0.8893669843673706,
+      "rewards/rejected": -1.9927412271499634,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 108.20955894474618,
+      "grad_norm": 112.04677171325864,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": 124.3506088256836,
-      "logits/rejected": 112.85099792480469,
-      "logps/chosen": -398.3126220703125,
-      "logps/rejected": -424.8102111816406,
-      "loss": 0.5433,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.205923080444336,
-      "rewards/margins": 1.1408246755599976,
-      "rewards/rejected": -3.346747636795044,
+      "logits/chosen": 124.56459045410156,
+      "logits/rejected": 113.08979797363281,
+      "logps/chosen": -391.10174560546875,
+      "logps/rejected": -417.09051513671875,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8466203212738037,
+      "rewards/margins": 1.0624934434890747,
+      "rewards/rejected": -2.909113883972168,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 126.72490253404305,
+      "grad_norm": 126.0267199667638,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": 139.5634002685547,
-      "logits/rejected": 143.65257263183594,
-      "logps/chosen": -460.2559509277344,
-      "logps/rejected": -542.6179809570312,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.055608034133911,
-      "rewards/margins": 1.258972406387329,
-      "rewards/rejected": -3.3145804405212402,
+      "logits/chosen": 138.824462890625,
+      "logits/rejected": 142.9259033203125,
+      "logps/chosen": -455.6646423339844,
+      "logps/rejected": -536.987548828125,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8371152877807617,
+      "rewards/margins": 1.2058273553848267,
+      "rewards/rejected": -3.042942523956299,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 61.8785357181442,
+      "grad_norm": 63.02753605606795,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": 131.92459106445312,
-      "logits/rejected": 135.47061157226562,
-      "logps/chosen": -446.0873107910156,
-      "logps/rejected": -530.6946411132812,
-      "loss": 0.3112,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -2.4040818214416504,
-      "rewards/margins": 2.312370777130127,
-      "rewards/rejected": -4.716452121734619,
+      "logits/chosen": 131.37802124023438,
+      "logits/rejected": 134.72222900390625,
+      "logps/chosen": -436.17047119140625,
+      "logps/rejected": -520.2355346679688,
+      "loss": 0.3078,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -1.9126123189926147,
+      "rewards/margins": 2.285891056060791,
+      "rewards/rejected": -4.198503017425537,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 41.57352214721893,
+      "grad_norm": 59.15589622996558,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": 124.61039733886719,
-      "logits/rejected": 126.63330078125,
-      "logps/chosen": -418.581787109375,
-      "logps/rejected": -522.1246337890625,
-      "loss": 0.1985,
+      "logits/chosen": 124.51689147949219,
+      "logits/rejected": 126.70524597167969,
+      "logps/chosen": -402.62066650390625,
+      "logps/rejected": -505.50006103515625,
+      "loss": 0.1932,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -2.5718960762023926,
-      "rewards/margins": 2.858640193939209,
-      "rewards/rejected": -5.43053674697876,
+      "rewards/chosen": -1.7549495697021484,
+      "rewards/margins": 2.8932533264160156,
+      "rewards/rejected": -4.648203372955322,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 45.00944046517108,
+      "grad_norm": 44.24206971141979,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": 110.86280822753906,
-      "logits/rejected": 127.59481048583984,
-      "logps/chosen": -417.392578125,
-      "logps/rejected": -536.7481689453125,
-      "loss": 0.1713,
+      "logits/chosen": 111.74947357177734,
+      "logits/rejected": 128.5332489013672,
+      "logps/chosen": -400.06146240234375,
+      "logps/rejected": -519.4473876953125,
+      "loss": 0.1687,
       "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -3.1870760917663574,
-      "rewards/margins": 2.988443374633789,
-      "rewards/rejected": -6.175518989562988,
+      "rewards/chosen": -2.326984167098999,
+      "rewards/margins": 2.937407970428467,
+      "rewards/rejected": -5.264392375946045,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 41.01677369762327,
+      "grad_norm": 50.07584592888485,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": 110.12068176269531,
-      "logits/rejected": 122.8779067993164,
-      "logps/chosen": -448.1509704589844,
-      "logps/rejected": -565.2049560546875,
-      "loss": 0.1566,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -3.085996150970459,
-      "rewards/margins": 3.2901604175567627,
-      "rewards/rejected": -6.376156330108643,
+      "logits/chosen": 110.84814453125,
+      "logits/rejected": 123.78230285644531,
+      "logps/chosen": -435.03265380859375,
+      "logps/rejected": -550.7723388671875,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.400338649749756,
+      "rewards/margins": 3.238767623901367,
+      "rewards/rejected": -5.639105796813965,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 39.730984591066736,
+      "grad_norm": 49.65552371508206,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": 116.5959701538086,
-      "logits/rejected": 116.32414245605469,
-      "logps/chosen": -436.4479064941406,
-      "logps/rejected": -531.0074462890625,
-      "loss": 0.1582,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -2.723642110824585,
-      "rewards/margins": 2.9885311126708984,
-      "rewards/rejected": -5.7121734619140625,
+      "logits/chosen": 117.1094970703125,
+      "logits/rejected": 117.060302734375,
+      "logps/chosen": -427.23431396484375,
+      "logps/rejected": -520.2066650390625,
+      "loss": 0.1591,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -2.2608113288879395,
+      "rewards/margins": 2.9113571643829346,
+      "rewards/rejected": -5.172169208526611,
       "step": 100
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": 96.0169448852539,
-      "eval_logits/rejected": 90.12943267822266,
-      "eval_logps/chosen": -434.5467224121094,
-      "eval_logps/rejected": -466.9059753417969,
-      "eval_loss": 0.44273361563682556,
+      "eval_logits/chosen": 96.76607513427734,
+      "eval_logits/rejected": 91.05736541748047,
+      "eval_logps/chosen": -422.7994689941406,
+      "eval_logps/rejected": -453.052978515625,
+      "eval_loss": 0.4691648781299591,
       "eval_rewards/accuracies": 0.7604166865348816,
-      "eval_rewards/chosen": -3.543926954269409,
-      "eval_rewards/margins": 1.6978217363357544,
-      "eval_rewards/rejected": -5.241748809814453,
-      "eval_runtime": 123.9778,
-      "eval_samples_per_second": 6.049,
-      "eval_steps_per_second": 0.194,
+      "eval_rewards/chosen": -2.9809672832489014,
+      "eval_rewards/margins": 1.6640973091125488,
+      "eval_rewards/rejected": -4.645064353942871,
+      "eval_runtime": 56.9732,
+      "eval_samples_per_second": 13.164,
+      "eval_steps_per_second": 0.421,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.391697224516135,
-      "train_runtime": 2305.723,
-      "train_samples_per_second": 5.855,
-      "train_steps_per_second": 0.045
+      "train_loss": 0.3921648321243433,
+      "train_runtime": 1190.3032,
+      "train_samples_per_second": 11.342,
+      "train_steps_per_second": 0.087
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32d4314fb42077b412ca888358e0197fb75b2357e9032ea0da1cd595a5e7608b
+oid sha256:912545168b662aa28a59d2ea7ff1a88af69cd74e0b822da195076b8d4f0f07e5
 size 6264