Training in progress, step 600

README.md

@@ -20,15 +20,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3778
-- Rewards/chosen: 0.1312
-- Rewards/rejected: 0.1259
-- Rewards/accuracies: 0.5166
-- Rewards/margins: 0.0052
-- Logps/rejected: -37.3767
-- Logps/chosen: -33.8888
-- Logits/rejected: -2.2423
-- Logits/chosen: -2.2472
+- Loss: 0.9894
+- Rewards/chosen: -0.0296
+- Rewards/rejected: -0.0401
+- Rewards/accuracies: 0.5125
+- Rewards/margins: 0.0105
+- Logps/rejected: -37.6168
+- Logps/chosen: -34.1085
+- Logits/rejected: -2.2185
+- Logits/chosen: -2.2233
 
 ## Model description
 
@@ -57,33 +57,21 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 4
+- num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.3038        | 0.26  | 100  | -2.2372       | -2.2324         | -34.0128     | -37.5115       | 0.3512          | 0.5424             | 0.0196         | 0.0150          | 0.0046           |
-| 0.3157        | 0.52  | 200  | -2.2371       | -2.2322         | -34.0181     | -37.5184       | 0.3716          | 0.5245             | 0.0148         | 0.0164          | -0.0016          |
-| 0.2156        | 0.78  | 300  | -2.2364       | -2.2316         | -34.0143     | -37.4970       | 0.3845          | 0.4934             | 0.0182         | 0.0005          | 0.0177           |
-| 0.4084        | 1.04  | 400  | 0.4059        | 0.0705          | 0.0718       | 0.5066         | -0.0013         | -37.4369           | -33.9562       | -2.2400         | -2.2448          |
-| 0.2788        | 1.3   | 500  | 0.3866        | 0.0701          | 0.0576       | 0.5191         | 0.0125          | -37.4526           | -33.9566       | -2.2356         | -2.2405          |
-| 0.3874        | 1.56  | 600  | 0.4265        | 0.0711          | 0.0890       | 0.4726         | -0.0180         | -37.4177           | -33.9556       | -2.2421         | -2.2470          |
-| 0.2695        | 1.82  | 700  | 0.4028        | 0.0816          | 0.0876       | 0.5079         | -0.0060         | -37.4193           | -33.9439       | -2.2429         | -2.2478          |
-| 0.1725        | 2.08  | 800  | 0.4083        | 0.0967          | 0.1077       | 0.4821         | -0.0110         | -37.3970           | -33.9271       | -2.2415         | -2.2463          |
-| 0.2502        | 2.34  | 900  | 0.4099        | 0.1154          | 0.1311       | 0.4900         | -0.0157         | -37.3709           | -33.9064       | -2.2438         | -2.2487          |
-| 0.1529        | 2.6   | 1000 | 0.3879        | 0.1222          | 0.1257       | 0.5216         | -0.0034         | -37.3770           | -33.8988       | -2.2428         | -2.2477          |
-| 0.1583        | 2.86  | 1100 | 0.3968        | 0.1193          | 0.1250       | 0.4875         | -0.0057         | -37.3777           | -33.9020       | -2.2433         | -2.2482          |
-| 0.113         | 3.12  | 1200 | 0.3849        | 0.1137          | 0.1163       | 0.4784         | -0.0025         | -37.3874           | -33.9082       | -2.2421         | -2.2470          |
-| 0.0937        | 3.38  | 1300 | 0.3738        | 0.1235          | 0.1177       | 0.5046         | 0.0058          | -37.3859           | -33.8973       | -2.2423         | -2.2472          |
-| 0.0815        | 3.64  | 1400 | 0.3595        | 0.1338          | 0.1197       | 0.5224         | 0.0141          | -37.3836           | -33.8859       | -2.2427         | -2.2476          |
-| 0.0757        | 3.9   | 1500 | 0.3543        | 0.1332          | 0.1192       | 0.5486         | 0.0139          | -37.3842           | -33.8866       | -2.2421         | -2.2469          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.9521        | 0.26  | 100  | 0.9920          | -0.0014        | -0.0094          | 0.5361             | 0.0080          | -37.5401       | -34.0381     | -2.2279         | -2.2327       |
+| 0.8515        | 0.52  | 200  | 0.9817          | -0.0042        | -0.0226          | 0.5594             | 0.0184          | -37.5730       | -34.0451     | -2.2226         | -2.2274       |
+| 0.7473        | 0.78  | 300  | 0.9830          | -0.0231        | -0.0399          | 0.5390             | 0.0169          | -37.6165       | -34.0922     | -2.2186         | -2.2234       |
 
 
 ### Framework versions
 
-- PEFT 0.10.0
-- Transformers 4.39.0.dev0
+- PEFT 0.8.2
+- Transformers 4.37.2
 - Pytorch 2.1.2+cu121
-- Datasets 2.14.6
+- Datasets 2.17.0
 - Tokenizers 0.15.1

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "o_proj",
+    "v_proj",
     "up_proj",
-    "down_proj",
-    "k_proj",
-    "gate_proj",
     "q_proj",
-    "v_proj"
+    "down_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6204b6d9975755eaf1dfc0eeb539db7dfda9602c6eb76efa388fc044fba516b9
+oid sha256:c9b00993b18b4e267fab497ab573e36b6c733a3766b440af653d912525328854
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 4.0,
-    "eval_logits/chosen": -2.2472212314605713,
-    "eval_logits/rejected": -2.24234938621521,
-    "eval_logps/chosen": -33.88882064819336,
-    "eval_logps/rejected": -37.376686096191406,
-    "eval_loss": 0.37784525752067566,
-    "eval_rewards/accuracies": 0.5166113376617432,
-    "eval_rewards/chosen": 0.13115935027599335,
-    "eval_rewards/margins": 0.0052182856015861034,
-    "eval_rewards/rejected": 0.12594106793403625,
-    "eval_runtime": 145.1836,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.223268747329712,
+    "eval_logits/rejected": -2.218452215194702,
+    "eval_logps/chosen": -34.1085205078125,
+    "eval_logps/rejected": -37.61680603027344,
+    "eval_loss": 0.9893841743469238,
+    "eval_rewards/accuracies": 0.5124584436416626,
+    "eval_rewards/chosen": -0.02958693355321884,
+    "eval_rewards/margins": 0.010487398132681847,
+    "eval_rewards/rejected": -0.04007433354854584,
+    "eval_runtime": 145.4187,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.363,
+    "eval_samples_per_second": 2.359,
     "eval_steps_per_second": 0.296,
-    "train_loss": 0.17404492969636792,
-    "train_runtime": 10767.1792,
+    "train_loss": 0.8990784768934373,
+    "train_runtime": 3250.3805,
     "train_samples": 3079,
-    "train_samples_per_second": 1.144,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

config.json

@@ -18,8 +18,6 @@
   "n_layer": 28,
   "n_positions": 2048,
   "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "float16",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": false,
@@ -47,7 +45,7 @@
   },
   "tie_word_embeddings": false,
   "tokenizer_class": "GPT2Tokenizer",
-  "transformers_version": "4.39.0.dev0",
+  "transformers_version": "4.37.2",
   "use_cache": true,
   "vocab_size": 50400
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 4.0,
-    "eval_logits/chosen": -2.2472212314605713,
-    "eval_logits/rejected": -2.24234938621521,
-    "eval_logps/chosen": -33.88882064819336,
-    "eval_logps/rejected": -37.376686096191406,
-    "eval_loss": 0.37784525752067566,
-    "eval_rewards/accuracies": 0.5166113376617432,
-    "eval_rewards/chosen": 0.13115935027599335,
-    "eval_rewards/margins": 0.0052182856015861034,
-    "eval_rewards/rejected": 0.12594106793403625,
-    "eval_runtime": 145.1836,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.223268747329712,
+    "eval_logits/rejected": -2.218452215194702,
+    "eval_logps/chosen": -34.1085205078125,
+    "eval_logps/rejected": -37.61680603027344,
+    "eval_loss": 0.9893841743469238,
+    "eval_rewards/accuracies": 0.5124584436416626,
+    "eval_rewards/chosen": -0.02958693355321884,
+    "eval_rewards/margins": 0.010487398132681847,
+    "eval_rewards/rejected": -0.04007433354854584,
+    "eval_runtime": 145.4187,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.363,
+    "eval_samples_per_second": 2.359,
     "eval_steps_per_second": 0.296
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 4.0,
-    "train_loss": 0.17404492969636792,
-    "train_runtime": 10767.1792,
+    "epoch": 1.0,
+    "train_loss": 0.8990784768934373,
+    "train_runtime": 3250.3805,
     "train_samples": 3079,
-    "train_samples_per_second": 1.144,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2afba1b889f3ad96a2cd4605a24926c4d3988731b470971ff3b87ea9f7f34337
+oid sha256:c5b701d7d845bc2558e4065820fdb6eb63d35a8aeb5ca845505877a84afc9d8d
 size 5176