Training in progress, step 600

README.md

@@ -20,15 +20,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9173
-- Rewards/chosen: -0.0874
-- Rewards/rejected: -0.1849
-- Rewards/accuracies: 0.5743
-- Rewards/margins: 0.0975
-- Logps/rejected: -37.7220
-- Logps/chosen: -34.1316
-- Logits/rejected: -2.2003
-- Logits/chosen: -2.2051
+- Loss: 0.6981
+- Rewards/chosen: -0.0558
+- Rewards/rejected: -0.0768
+- Rewards/accuracies: 0.5453
+- Rewards/margins: 0.0210
+- Logps/rejected: -37.6020
+- Logps/chosen: -34.0965
+- Logits/rejected: -2.2248
+- Logits/chosen: -2.2297
 
 ## Model description
 
@@ -57,33 +57,21 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 4
+- num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.8913        | 0.26  | 100  | -2.2322       | -2.2273         | -34.0407     | -37.5405       | 0.9845          | 0.5195             | -0.0055        | 0.0159          | -0.0215          |
-| 0.7293        | 0.52  | 200  | -2.2286       | -2.2238         | -34.0537     | -37.5811       | 0.9602          | 0.5714             | -0.0172        | 0.0408          | -0.0580          |
-| 0.6144        | 0.78  | 300  | -2.2249       | -2.2201         | -34.0866     | -37.6032       | 0.9713          | 0.5282             | -0.0468        | 0.0310          | -0.0779          |
-| 0.3632        | 1.04  | 400  | 0.9495        | -0.0909         | -0.1434      | 0.5602         | 0.0525          | -37.6760           | -34.1355       | -2.2076         | -2.2125          |
-| 0.2994        | 1.3   | 500  | 0.9461        | -0.1647         | -0.2318      | 0.5540         | 0.0671          | -37.7742           | -34.2176       | -2.2162         | -2.2210          |
-| 0.3408        | 1.56  | 600  | 0.9077        | -0.0675         | -0.1694      | 0.5868         | 0.1019          | -37.7048           | -34.1096       | -2.2017         | -2.2066          |
-| 0.2796        | 1.82  | 700  | 0.9425        | -0.0929         | -0.1626      | 0.5569         | 0.0697          | -37.6973           | -34.1378       | -2.2012         | -2.2061          |
-| 0.1052        | 2.08  | 800  | 0.9125        | -0.0848         | -0.1863      | 0.5926         | 0.1015          | -37.7236           | -34.1288       | -2.2003         | -2.2051          |
-| 0.095         | 2.34  | 900  | 0.9005        | -0.0802         | -0.1942      | 0.5540         | 0.1140          | -37.7324           | -34.1237       | -2.2019         | -2.2067          |
-| 0.123         | 2.6   | 1000 | 0.9194        | -0.0907         | -0.1876      | 0.5511         | 0.0969          | -37.7251           | -34.1353       | -2.1994         | -2.2043          |
-| 0.0894        | 2.86  | 1100 | 0.9182        | -0.0915         | -0.1890      | 0.5336         | 0.0976          | -37.7267           | -34.1362       | -2.2001         | -2.2050          |
-| 0.1086        | 3.12  | 1200 | 0.9023        | -0.0864         | -0.1976      | 0.5627         | 0.1112          | -37.7362           | -34.1306       | -2.2006         | -2.2054          |
-| 0.0577        | 3.38  | 1300 | 0.9154        | -0.0922         | -0.1935      | 0.5598         | 0.1013          | -37.7317           | -34.1370       | -2.2002         | -2.2050          |
-| 0.0375        | 3.64  | 1400 | 0.9233        | -0.0896         | -0.1810      | 0.5569         | 0.0914          | -37.7178           | -34.1342       | -2.2002         | -2.2050          |
-| 0.0724        | 3.9   | 1500 | 0.9176        | -0.0906         | -0.1873      | 0.5627         | 0.0967          | -37.7248           | -34.1353       | -2.2000         | -2.2049          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6464        | 0.26  | 100  | 0.6903          | -0.0054        | -0.0300          | 0.5685             | 0.0246          | -37.5500       | -34.0405     | -2.2291         | -2.2340       |
+| 0.5931        | 0.52  | 200  | 0.6980          | -0.0346        | -0.0543          | 0.5158             | 0.0196          | -37.5769       | -34.0730     | -2.2267         | -2.2316       |
+| 0.5301        | 0.78  | 300  | 0.6973          | -0.0555        | -0.0750          | 0.5390             | 0.0195          | -37.6000       | -34.0962     | -2.2243         | -2.2292       |
 
 
 ### Framework versions
 
-- PEFT 0.10.0
-- Transformers 4.39.0.dev0
+- PEFT 0.8.2
+- Transformers 4.37.2
 - Pytorch 2.1.2+cu121
-- Datasets 2.14.6
+- Datasets 2.17.0
 - Tokenizers 0.15.1

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
     "q_proj",
-    "up_proj",
-    "k_proj",
-    "o_proj",
     "v_proj",
+    "k_proj",
     "down_proj",
-    "gate_proj"
+    "up_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91d910fca51cc8ac20bb694b76c7d445531ace7d2e5055536929a38bcbb377dd
+oid sha256:e7a780872ae0ab102acf92ea1710d8aece0f65a8a4c151c4577f63501c468697
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 4.0,
-    "eval_logits/chosen": -2.2051267623901367,
-    "eval_logits/rejected": -2.2002947330474854,
-    "eval_logps/chosen": -34.13163375854492,
-    "eval_logps/rejected": -37.722042083740234,
-    "eval_loss": 0.9173061847686768,
-    "eval_rewards/accuracies": 0.574335515499115,
-    "eval_rewards/chosen": -0.08737529814243317,
-    "eval_rewards/margins": 0.09750958532094955,
-    "eval_rewards/rejected": -0.18488489091396332,
-    "eval_runtime": 145.0469,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.22967529296875,
+    "eval_logits/rejected": -2.2248213291168213,
+    "eval_logps/chosen": -34.096519470214844,
+    "eval_logps/rejected": -37.601966857910156,
+    "eval_loss": 0.6981170177459717,
+    "eval_rewards/accuracies": 0.545265793800354,
+    "eval_rewards/chosen": -0.0557732917368412,
+    "eval_rewards/margins": 0.021040165796875954,
+    "eval_rewards/rejected": -0.07681345194578171,
+    "eval_runtime": 145.6397,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.365,
-    "eval_steps_per_second": 0.296,
-    "train_loss": 0.15107146371881683,
-    "train_runtime": 10740.8737,
+    "eval_samples_per_second": 2.355,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.6175476637753573,
+    "train_runtime": 3252.7839,
     "train_samples": 3079,
-    "train_samples_per_second": 1.147,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

config.json

@@ -18,8 +18,6 @@
   "n_layer": 28,
   "n_positions": 2048,
   "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "float16",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": false,
@@ -47,7 +45,7 @@
   },
   "tie_word_embeddings": false,
   "tokenizer_class": "GPT2Tokenizer",
-  "transformers_version": "4.39.0.dev0",
+  "transformers_version": "4.37.2",
   "use_cache": true,
   "vocab_size": 50400
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 4.0,
-    "eval_logits/chosen": -2.2051267623901367,
-    "eval_logits/rejected": -2.2002947330474854,
-    "eval_logps/chosen": -34.13163375854492,
-    "eval_logps/rejected": -37.722042083740234,
-    "eval_loss": 0.9173061847686768,
-    "eval_rewards/accuracies": 0.574335515499115,
-    "eval_rewards/chosen": -0.08737529814243317,
-    "eval_rewards/margins": 0.09750958532094955,
-    "eval_rewards/rejected": -0.18488489091396332,
-    "eval_runtime": 145.0469,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.22967529296875,
+    "eval_logits/rejected": -2.2248213291168213,
+    "eval_logps/chosen": -34.096519470214844,
+    "eval_logps/rejected": -37.601966857910156,
+    "eval_loss": 0.6981170177459717,
+    "eval_rewards/accuracies": 0.545265793800354,
+    "eval_rewards/chosen": -0.0557732917368412,
+    "eval_rewards/margins": 0.021040165796875954,
+    "eval_rewards/rejected": -0.07681345194578171,
+    "eval_runtime": 145.6397,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.365,
-    "eval_steps_per_second": 0.296
+    "eval_samples_per_second": 2.355,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 4.0,
-    "train_loss": 0.15107146371881683,
-    "train_runtime": 10740.8737,
+    "epoch": 1.0,
+    "train_loss": 0.6175476637753573,
+    "train_runtime": 3252.7839,
     "train_samples": 3079,
-    "train_samples_per_second": 1.147,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b693805000bf6ee72fa3619058beec5d399922cc80612ea5e9a7a7feb2374ed
+oid sha256:bdf4888b081a55a294000817bce0516657ba047dfd048058409647c84bbb0cd8
 size 5176