third model

README.md

@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [stabilityai/stablelm-3b-4e1t](https://huggingface.co/stabilityai/stablelm-3b-4e1t) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.2700
+- Loss: 4.0159
 
 ## Model description
 
@@ -37,9 +37,9 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0001
-- train_batch_size: 32
-- eval_batch_size: 32
+- learning_rate: 0.0005
+- train_batch_size: 16
+- eval_batch_size: 16
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -49,11 +49,17 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 4.7338        | 0.18  | 100  | 4.5703          |
-| 4.6391        | 0.36  | 200  | 4.4094          |
-| 4.8127        | 0.54  | 300  | 4.3277          |
-| 4.5393        | 0.72  | 400  | 4.2797          |
-| 4.29          | 0.9   | 500  | 4.2700          |
+| 4.6872        | 0.09  | 100  | 4.5976          |
+| 4.1956        | 0.18  | 200  | 4.4367          |
+| 4.3439        | 0.27  | 300  | 4.3083          |
+| 4.3709        | 0.36  | 400  | 4.2462          |
+| 4.3886        | 0.45  | 500  | 4.1890          |
+| 4.402         | 0.54  | 600  | 4.1248          |
+| 4.043         | 0.63  | 700  | 4.0840          |
+| 4.1489        | 0.72  | 800  | 4.0423          |
+| 4.1139        | 0.81  | 900  | 4.0251          |
+| 4.273         | 0.9   | 1000 | 4.0162          |
+| 3.8663        | 1.0   | 1100 | 4.0159          |
 
 
 ### Framework versions

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "stabilityai/stablelm-3b-4e1t",
+  "base_model_name_or_path": null,
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -15,235 +15,235 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 256,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "layers.13.self_attn.o_proj",
-    "layers.27.mlp.up_proj",
-    "layers.31.self_attn.q_proj",
-    "layers.24.self_attn.q_proj",
-    "layers.10.self_attn.v_proj",
-    "layers.24.mlp.down_proj",
-    "layers.22.mlp.down_proj",
-    "layers.4.self_attn.q_proj",
-    "layers.25.mlp.down_proj",
-    "layers.1.mlp.up_proj",
-    "layers.10.mlp.gate_proj",
-    "layers.0.self_attn.v_proj",
+    "layers.16.self_attn.k_proj",
+    "layers.27.mlp.gate_proj",
+    "layers.31.mlp.up_proj",
+    "layers.5.mlp.up_proj",
+    "layers.4.self_attn.o_proj",
     "layers.18.mlp.gate_proj",
-    "layers.22.mlp.gate_proj",
-    "layers.3.self_attn.q_proj",
-    "layers.11.mlp.gate_proj",
-    "layers.21.self_attn.q_proj",
-    "layers.8.mlp.gate_proj",
-    "layers.14.self_attn.q_proj",
-    "layers.28.mlp.gate_proj",
-    "layers.1.self_attn.v_proj",
-    "layers.15.mlp.up_proj",
-    "layers.25.self_attn.v_proj",
+    "layers.23.mlp.down_proj",
+    "layers.3.mlp.down_proj",
     "layers.23.self_attn.o_proj",
-    "layers.19.mlp.gate_proj",
-    "layers.1.self_attn.k_proj",
-    "layers.1.mlp.down_proj",
-    "layers.13.mlp.down_proj",
-    "layers.23.mlp.up_proj",
-    "layers.6.self_attn.o_proj",
-    "layers.27.self_attn.v_proj",
-    "layers.13.self_attn.k_proj",
-    "layers.11.mlp.up_proj",
-    "layers.5.mlp.up_proj",
-    "layers.29.mlp.down_proj",
-    "layers.30.self_attn.v_proj",
-    "layers.9.self_attn.o_proj",
-    "layers.15.self_attn.q_proj",
-    "layers.25.self_attn.k_proj",
-    "layers.17.self_attn.v_proj",
-    "layers.23.self_attn.v_proj",
-    "layers.12.self_attn.k_proj",
     "layers.31.self_attn.o_proj",
-    "layers.21.mlp.gate_proj",
-    "layers.2.mlp.up_proj",
-    "layers.9.mlp.up_proj",
-    "layers.10.self_attn.o_proj",
-    "layers.0.mlp.down_proj",
-    "layers.3.mlp.gate_proj",
-    "layers.5.mlp.down_proj",
-    "layers.9.mlp.down_proj",
-    "layers.12.mlp.up_proj",
-    "layers.0.self_attn.k_proj",
-    "layers.25.self_attn.o_proj",
-    "layers.24.mlp.up_proj",
-    "layers.26.mlp.up_proj",
-    "layers.20.self_attn.v_proj",
-    "layers.31.self_attn.v_proj",
-    "layers.0.self_attn.q_proj",
-    "layers.30.mlp.gate_proj",
-    "layers.14.self_attn.o_proj",
-    "layers.8.self_attn.v_proj",
+    "layers.6.mlp.gate_proj",
+    "layers.11.self_attn.k_proj",
+    "layers.8.self_attn.o_proj",
+    "layers.29.self_attn.o_proj",
+    "layers.3.self_attn.q_proj",
+    "layers.4.self_attn.q_proj",
     "layers.29.mlp.gate_proj",
-    "layers.6.mlp.down_proj",
-    "layers.16.mlp.down_proj",
-    "layers.22.self_attn.q_proj",
-    "layers.28.self_attn.v_proj",
+    "layers.19.self_attn.q_proj",
+    "layers.16.mlp.gate_proj",
+    "layers.13.mlp.up_proj",
+    "layers.29.self_attn.k_proj",
+    "layers.0.self_attn.o_proj",
+    "layers.17.mlp.down_proj",
+    "layers.25.mlp.up_proj",
+    "layers.19.self_attn.v_proj",
+    "layers.6.self_attn.q_proj",
+    "layers.5.self_attn.o_proj",
+    "layers.11.mlp.up_proj",
+    "layers.12.self_attn.q_proj",
+    "layers.24.self_attn.k_proj",
+    "layers.8.self_attn.v_proj",
+    "layers.31.self_attn.k_proj",
+    "layers.26.mlp.down_proj",
     "layers.31.mlp.gate_proj",
+    "layers.20.self_attn.v_proj",
+    "layers.11.self_attn.q_proj",
+    "layers.19.mlp.up_proj",
+    "layers.13.self_attn.o_proj",
     "layers.4.self_attn.k_proj",
-    "layers.12.self_attn.o_proj",
-    "layers.10.mlp.up_proj",
-    "layers.19.self_attn.q_proj",
-    "layers.8.self_attn.o_proj",
-    "layers.16.self_attn.o_proj",
-    "layers.3.mlp.down_proj",
-    "layers.26.self_attn.k_proj",
-    "layers.7.mlp.down_proj",
-    "layers.10.self_attn.k_proj",
-    "layers.23.self_attn.k_proj",
-    "layers.21.self_attn.o_proj",
-    "layers.18.self_attn.k_proj",
-    "layers.27.mlp.down_proj",
+    "layers.11.self_attn.o_proj",
+    "layers.4.mlp.gate_proj",
+    "layers.15.self_attn.q_proj",
     "layers.17.self_attn.o_proj",
-    "layers.21.self_attn.k_proj",
-    "layers.30.mlp.down_proj",
-    "layers.3.self_attn.k_proj",
-    "layers.25.mlp.gate_proj",
-    "layers.16.mlp.gate_proj",
+    "layers.6.self_attn.k_proj",
+    "layers.12.mlp.down_proj",
+    "layers.9.self_attn.k_proj",
+    "layers.17.mlp.up_proj",
+    "layers.1.mlp.gate_proj",
+    "layers.20.mlp.gate_proj",
+    "layers.4.self_attn.v_proj",
+    "layers.16.self_attn.v_proj",
     "layers.31.mlp.down_proj",
-    "layers.22.self_attn.o_proj",
-    "layers.6.mlp.gate_proj",
-    "layers.11.mlp.down_proj",
-    "layers.23.self_attn.q_proj",
-    "layers.7.mlp.up_proj",
-    "layers.22.self_attn.k_proj",
-    "layers.12.self_attn.q_proj",
-    "layers.19.self_attn.v_proj",
-    "layers.22.self_attn.v_proj",
-    "layers.24.mlp.gate_proj",
-    "layers.2.self_attn.q_proj",
-    "layers.2.mlp.gate_proj",
-    "layers.30.self_attn.q_proj",
-    "layers.14.self_attn.v_proj",
-    "layers.26.self_attn.v_proj",
-    "layers.28.self_attn.q_proj",
-    "layers.30.mlp.up_proj",
-    "layers.10.mlp.down_proj",
-    "layers.5.self_attn.q_proj",
-    "layers.21.self_attn.v_proj",
-    "layers.15.self_attn.k_proj",
-    "layers.1.self_attn.o_proj",
-    "layers.20.self_attn.k_proj",
-    "layers.4.self_attn.o_proj",
-    "layers.7.self_attn.k_proj",
-    "layers.14.self_attn.k_proj",
-    "layers.12.self_attn.v_proj",
-    "layers.16.self_attn.q_proj",
-    "layers.28.self_attn.o_proj",
-    "layers.29.mlp.up_proj",
-    "layers.18.self_attn.o_proj",
+    "layers.29.self_attn.v_proj",
+    "layers.26.self_attn.o_proj",
+    "layers.8.self_attn.q_proj",
+    "layers.10.self_attn.q_proj",
+    "layers.27.self_attn.o_proj",
     "layers.8.mlp.down_proj",
-    "layers.9.self_attn.q_proj",
-    "layers.25.mlp.up_proj",
-    "layers.14.mlp.gate_proj",
-    "layers.11.self_attn.v_proj",
+    "layers.29.mlp.down_proj",
     "layers.18.mlp.up_proj",
-    "layers.15.self_attn.o_proj",
-    "embed_tokens",
-    "layers.5.mlp.gate_proj",
-    "layers.29.self_attn.o_proj",
-    "layers.16.mlp.up_proj",
-    "layers.27.self_attn.k_proj",
-    "layers.26.self_attn.q_proj",
-    "layers.20.mlp.down_proj",
-    "layers.4.mlp.up_proj",
-    "layers.19.mlp.up_proj",
+    "layers.28.self_attn.k_proj",
+    "layers.27.self_attn.v_proj",
+    "layers.29.mlp.up_proj",
+    "layers.2.self_attn.q_proj",
+    "layers.21.self_attn.o_proj",
+    "layers.23.self_attn.k_proj",
+    "layers.28.self_attn.v_proj",
+    "layers.15.mlp.up_proj",
     "layers.18.mlp.down_proj",
-    "layers.3.self_attn.o_proj",
-    "layers.8.mlp.up_proj",
-    "layers.8.self_attn.k_proj",
-    "layers.30.self_attn.k_proj",
-    "layers.15.mlp.down_proj",
-    "layers.10.self_attn.q_proj",
-    "layers.16.self_attn.v_proj",
-    "layers.12.mlp.gate_proj",
     "layers.6.mlp.up_proj",
-    "layers.0.mlp.up_proj",
-    "layers.9.self_attn.k_proj",
-    "layers.31.self_attn.k_proj",
-    "layers.20.mlp.up_proj",
-    "layers.28.mlp.down_proj",
-    "layers.0.self_attn.o_proj",
-    "layers.4.mlp.gate_proj",
-    "layers.21.mlp.up_proj",
-    "layers.13.self_attn.q_proj",
-    "layers.5.self_attn.k_proj",
-    "layers.21.mlp.down_proj",
-    "layers.17.mlp.down_proj",
+    "layers.24.mlp.gate_proj",
+    "layers.24.self_attn.o_proj",
+    "layers.7.mlp.gate_proj",
+    "layers.12.self_attn.v_proj",
+    "layers.9.mlp.gate_proj",
+    "layers.4.mlp.up_proj",
     "layers.30.self_attn.o_proj",
+    "layers.28.mlp.gate_proj",
     "layers.19.mlp.down_proj",
-    "layers.1.mlp.gate_proj",
-    "layers.9.mlp.gate_proj",
-    "layers.13.mlp.gate_proj",
+    "layers.31.self_attn.q_proj",
+    "layers.5.self_attn.q_proj",
+    "layers.1.self_attn.o_proj",
+    "layers.25.self_attn.k_proj",
+    "layers.10.self_attn.k_proj",
+    "layers.12.self_attn.o_proj",
+    "layers.28.mlp.down_proj",
+    "layers.15.mlp.down_proj",
+    "layers.23.mlp.up_proj",
+    "layers.26.mlp.gate_proj",
     "layers.19.self_attn.k_proj",
-    "layers.3.self_attn.v_proj",
-    "layers.20.mlp.gate_proj",
+    "layers.14.mlp.up_proj",
+    "layers.20.self_attn.q_proj",
+    "layers.26.self_attn.q_proj",
+    "layers.0.mlp.down_proj",
+    "layers.21.self_attn.v_proj",
+    "layers.10.mlp.down_proj",
+    "layers.21.self_attn.k_proj",
+    "layers.3.self_attn.o_proj",
+    "layers.3.mlp.up_proj",
+    "layers.30.self_attn.q_proj",
+    "layers.6.mlp.down_proj",
+    "layers.9.mlp.down_proj",
+    "layers.8.self_attn.k_proj",
+    "layers.17.self_attn.q_proj",
+    "layers.11.mlp.gate_proj",
+    "layers.13.mlp.down_proj",
+    "layers.23.self_attn.q_proj",
+    "layers.9.self_attn.q_proj",
+    "layers.21.self_attn.q_proj",
     "layers.7.self_attn.v_proj",
-    "layers.19.self_attn.o_proj",
-    "layers.22.mlp.up_proj",
+    "layers.0.mlp.up_proj",
+    "layers.16.mlp.down_proj",
     "layers.17.self_attn.k_proj",
-    "layers.6.self_attn.q_proj",
-    "layers.24.self_attn.o_proj",
-    "layers.16.self_attn.k_proj",
-    "layers.13.mlp.up_proj",
-    "layers.8.self_attn.q_proj",
-    "layers.29.self_attn.k_proj",
-    "layers.28.self_attn.k_proj",
-    "layers.27.mlp.gate_proj",
-    "layers.2.self_attn.o_proj",
-    "layers.28.mlp.up_proj",
+    "layers.20.self_attn.o_proj",
+    "layers.3.mlp.gate_proj",
     "layers.25.self_attn.q_proj",
-    "layers.23.mlp.gate_proj",
-    "layers.9.self_attn.v_proj",
-    "layers.7.mlp.gate_proj",
-    "layers.17.mlp.up_proj",
-    "layers.5.self_attn.v_proj",
-    "layers.7.self_attn.o_proj",
+    "layers.6.self_attn.v_proj",
     "layers.13.self_attn.v_proj",
-    "layers.27.self_attn.q_proj",
-    "layers.18.self_attn.q_proj",
-    "layers.2.self_attn.v_proj",
-    "layers.23.mlp.down_proj",
-    "layers.14.mlp.down_proj",
+    "layers.7.self_attn.o_proj",
+    "layers.11.mlp.down_proj",
+    "layers.24.mlp.up_proj",
+    "layers.14.self_attn.k_proj",
+    "layers.0.mlp.gate_proj",
+    "layers.22.self_attn.o_proj",
+    "layers.12.mlp.gate_proj",
+    "layers.20.mlp.up_proj",
+    "layers.0.self_attn.v_proj",
+    "layers.25.self_attn.v_proj",
+    "layers.31.self_attn.v_proj",
+    "layers.22.self_attn.q_proj",
+    "layers.23.self_attn.v_proj",
+    "layers.9.self_attn.o_proj",
+    "layers.1.self_attn.q_proj",
+    "layers.0.self_attn.q_proj",
+    "layers.1.self_attn.v_proj",
+    "layers.28.self_attn.o_proj",
+    "layers.1.mlp.down_proj",
+    "layers.25.mlp.down_proj",
+    "layers.16.self_attn.o_proj",
+    "layers.16.mlp.up_proj",
+    "layers.22.self_attn.k_proj",
+    "layers.12.mlp.up_proj",
+    "layers.30.self_attn.k_proj",
+    "layers.14.self_attn.v_proj",
+    "layers.5.mlp.gate_proj",
+    "layers.18.self_attn.k_proj",
+    "layers.30.mlp.down_proj",
     "layers.2.self_attn.k_proj",
-    "layers.4.self_attn.v_proj",
-    "layers.26.self_attn.o_proj",
-    "layers.12.mlp.down_proj",
+    "layers.17.self_attn.v_proj",
+    "layers.13.mlp.gate_proj",
+    "layers.22.mlp.gate_proj",
+    "layers.10.self_attn.v_proj",
+    "layers.27.mlp.up_proj",
+    "layers.11.self_attn.v_proj",
+    "layers.2.self_attn.v_proj",
+    "layers.2.mlp.gate_proj",
+    "layers.1.self_attn.k_proj",
+    "layers.3.self_attn.v_proj",
+    "layers.21.mlp.gate_proj",
+    "layers.7.mlp.up_proj",
+    "layers.22.mlp.up_proj",
     "layers.7.self_attn.q_proj",
-    "layers.11.self_attn.k_proj",
-    "layers.27.self_attn.o_proj",
-    "layers.26.mlp.down_proj",
-    "layers.6.self_attn.v_proj",
-    "layers.4.mlp.down_proj",
-    "layers.11.self_attn.o_proj",
-    "layers.29.self_attn.q_proj",
-    "layers.24.self_attn.k_proj",
-    "layers.18.self_attn.v_proj",
-    "layers.20.self_attn.o_proj",
+    "layers.15.self_attn.v_proj",
+    "layers.27.self_attn.q_proj",
+    "layers.7.mlp.down_proj",
+    "layers.0.self_attn.k_proj",
+    "layers.20.mlp.down_proj",
+    "layers.28.mlp.up_proj",
+    "layers.30.self_attn.v_proj",
     "layers.2.mlp.down_proj",
-    "layers.11.self_attn.q_proj",
-    "layers.0.mlp.gate_proj",
-    "layers.3.mlp.up_proj",
-    "layers.6.self_attn.k_proj",
-    "layers.14.mlp.up_proj",
+    "layers.6.self_attn.o_proj",
+    "layers.15.self_attn.k_proj",
+    "layers.18.self_attn.o_proj",
+    "layers.8.mlp.up_proj",
+    "layers.4.mlp.down_proj",
+    "layers.14.mlp.gate_proj",
     "layers.17.mlp.gate_proj",
-    "layers.1.self_attn.q_proj",
+    "layers.14.self_attn.q_proj",
+    "layers.15.self_attn.o_proj",
+    "layers.26.self_attn.k_proj",
+    "layers.24.mlp.down_proj",
+    "layers.22.self_attn.v_proj",
+    "layers.18.self_attn.q_proj",
+    "layers.9.mlp.up_proj",
+    "layers.25.self_attn.o_proj",
+    "layers.9.self_attn.v_proj",
+    "layers.13.self_attn.k_proj",
+    "layers.20.self_attn.k_proj",
     "layers.24.self_attn.v_proj",
-    "layers.17.self_attn.q_proj",
-    "layers.20.self_attn.q_proj",
-    "layers.29.self_attn.v_proj",
-    "layers.5.self_attn.o_proj",
-    "layers.15.self_attn.v_proj",
+    "layers.24.self_attn.q_proj",
+    "layers.26.self_attn.v_proj",
+    "layers.2.self_attn.o_proj",
+    "layers.19.self_attn.o_proj",
+    "layers.30.mlp.gate_proj",
+    "layers.26.mlp.up_proj",
+    "layers.30.mlp.up_proj",
+    "layers.5.mlp.down_proj",
+    "layers.10.mlp.gate_proj",
+    "layers.12.self_attn.k_proj",
+    "layers.13.self_attn.q_proj",
     "layers.15.mlp.gate_proj",
-    "layers.31.mlp.up_proj",
-    "layers.26.mlp.gate_proj"
+    "layers.14.self_attn.o_proj",
+    "layers.25.mlp.gate_proj",
+    "layers.8.mlp.gate_proj",
+    "embed_tokens",
+    "layers.23.mlp.gate_proj",
+    "layers.2.mlp.up_proj",
+    "layers.27.mlp.down_proj",
+    "layers.10.mlp.up_proj",
+    "layers.3.self_attn.k_proj",
+    "layers.14.mlp.down_proj",
+    "layers.5.self_attn.k_proj",
+    "layers.21.mlp.up_proj",
+    "layers.22.mlp.down_proj",
+    "layers.27.self_attn.k_proj",
+    "layers.16.self_attn.q_proj",
+    "layers.5.self_attn.v_proj",
+    "layers.28.self_attn.q_proj",
+    "layers.21.mlp.down_proj",
+    "layers.10.self_attn.o_proj",
+    "layers.19.mlp.gate_proj",
+    "layers.7.self_attn.k_proj",
+    "layers.18.self_attn.v_proj",
+    "layers.1.mlp.up_proj",
+    "layers.29.self_attn.q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d716d04cda12d4d345b4c8b1c4e801a7682742bc807c90e463aa91a4f2925fcc
-size 421083096
+oid sha256:3f5ea8aceec5bb2e876d37cf17972fc88dc57ebab0b1291b47a465a3a0e5480f
+size 1039036344

runs/Feb02_14-21-05_eb570fa975b5/events.out.tfevents.1706883669.eb570fa975b5.1057.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fbab0603f15c8061d6d1fc77d71d56120c8cc4001764545b475da76cc1d2923
+size 181067

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1268c71a97ae93de424ebbbb07d10c6d8a2ff3ac001b287b75b1a9aed450f25f
+oid sha256:f519084678570cd5032dc5aa943f6f8ae827cf55a2e6c4d12e27323933e12fd8
 size 4728