Model save

README.md

@@ -2,9 +2,8 @@
 license: apache-2.0
 library_name: peft
 tags:
-- llama-factory
-- lora
 - unsloth
+- llama-factory
 - generated_from_trainer
 base_model: unsloth/llama-3-8b-Instruct-bnb-4bit
 model-index:
@@ -17,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # llama-3-8b-finetuned
 
-This model is a fine-tuned version of [unsloth/llama-3-8b-Instruct-bnb-4bit](https://huggingface.co/unsloth/llama-3-8b-Instruct-bnb-4bit) on the formatted_data dataset.
+This model is a fine-tuned version of [unsloth/llama-3-8b-Instruct-bnb-4bit](https://huggingface.co/unsloth/llama-3-8b-Instruct-bnb-4bit) on an unknown dataset.
 
 ## Model description
 
@@ -40,6 +39,7 @@ The following hyperparameters were used during training:
 - train_batch_size: 1
 - eval_batch_size: 8
 - seed: 42
+- distributed_type: multi-GPU
 - gradient_accumulation_steps: 4
 - total_train_batch_size: 4
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
@@ -56,6 +56,6 @@ The following hyperparameters were used during training:
 
 - PEFT 0.11.1
 - Transformers 4.41.2
-- Pytorch 2.3.0+cu121
+- Pytorch 2.2.0+cu121
 - Datasets 2.19.2
 - Tokenizers 0.19.1

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": "unsloth",
   "target_modules": [
-    "gate_proj",
+    "v_proj",
     "o_proj",
-    "down_proj",
+    "gate_proj",
+    "q_proj",
     "k_proj",
     "up_proj",
-    "q_proj",
-    "v_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c65529b4d6a3ba2634f2ec48033308cf00df957936762c26d39a2a1a5e6e0b8
+oid sha256:69a24fca18d50118ce689d4dc26862aea2c95ebdbb5e8d3689fa569ebaaefee8
 size 83945296

runs/Jun10_16-50-08_jupyter-sze-2eong-2e23-40ucl-2eac-2euk/events.out.tfevents.1718034664.jupyter-sze-2eong-2e23-40ucl-2eac-2euk.1691.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d06b23a391a5f8f0f4819c6c523be75fcaa1b0f657d078336e9ed9890d0bc08
+size 4184

runs/Jun10_16-57-36_jupyter-sze-2eong-2e23-40ucl-2eac-2euk/events.out.tfevents.1718035086.jupyter-sze-2eong-2e23-40ucl-2eac-2euk.2073.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c380e95deb378e67a4cf676724a17efbac1ff9e7da79eac9057002b604fd1a2
+size 6974

runs/Jun10_17-05-22_jupyter-sze-2eong-2e23-40ucl-2eac-2euk/events.out.tfevents.1718035553.jupyter-sze-2eong-2e23-40ucl-2eac-2euk.2186.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:843d7d0b9efb5ae1cd08d4f51e53ed7a275993cf623a02e0a7920269f69fdbd9
+size 11106

trainer_log.jsonl

@@ -1,26 +1,26 @@
-{"current_steps": 10, "total_steps": 250, "loss": 1.585, "learning_rate": 7.2000000000000005e-06, "epoch": 0.08, "percentage": 4.0, "elapsed_time": "0:01:51", "remaining_time": "0:44:47"}
-{"current_steps": 20, "total_steps": 250, "loss": 0.268, "learning_rate": 1.5200000000000002e-05, "epoch": 0.16, "percentage": 8.0, "elapsed_time": "0:03:39", "remaining_time": "0:42:00"}
-{"current_steps": 30, "total_steps": 250, "loss": 0.2895, "learning_rate": 1.9984407641819812e-05, "epoch": 0.24, "percentage": 12.0, "elapsed_time": "0:05:26", "remaining_time": "0:39:54"}
-{"current_steps": 40, "total_steps": 250, "loss": 0.1859, "learning_rate": 1.9809551553491918e-05, "epoch": 0.32, "percentage": 16.0, "elapsed_time": "0:07:13", "remaining_time": "0:37:57"}
-{"current_steps": 50, "total_steps": 250, "loss": 0.0984, "learning_rate": 1.944376370237481e-05, "epoch": 0.4, "percentage": 20.0, "elapsed_time": "0:09:01", "remaining_time": "0:36:04"}
-{"current_steps": 60, "total_steps": 250, "loss": 0.0668, "learning_rate": 1.889416373291298e-05, "epoch": 0.48, "percentage": 24.0, "elapsed_time": "0:10:47", "remaining_time": "0:34:11"}
-{"current_steps": 70, "total_steps": 250, "loss": 0.1228, "learning_rate": 1.8171448983351284e-05, "epoch": 0.56, "percentage": 28.0, "elapsed_time": "0:12:34", "remaining_time": "0:32:20"}
-{"current_steps": 80, "total_steps": 250, "loss": 0.193, "learning_rate": 1.7289686274214116e-05, "epoch": 0.64, "percentage": 32.0, "elapsed_time": "0:14:21", "remaining_time": "0:30:31"}
-{"current_steps": 90, "total_steps": 250, "loss": 0.1573, "learning_rate": 1.6266038113644605e-05, "epoch": 0.72, "percentage": 36.0, "elapsed_time": "0:16:09", "remaining_time": "0:28:42"}
-{"current_steps": 100, "total_steps": 250, "loss": 0.0905, "learning_rate": 1.5120428648705716e-05, "epoch": 0.8, "percentage": 40.0, "elapsed_time": "0:17:56", "remaining_time": "0:26:54"}
-{"current_steps": 110, "total_steps": 250, "loss": 0.1169, "learning_rate": 1.3875155864521031e-05, "epoch": 0.88, "percentage": 44.0, "elapsed_time": "0:19:43", "remaining_time": "0:25:06"}
-{"current_steps": 120, "total_steps": 250, "loss": 0.125, "learning_rate": 1.2554457579357906e-05, "epoch": 0.96, "percentage": 48.0, "elapsed_time": "0:21:30", "remaining_time": "0:23:18"}
-{"current_steps": 130, "total_steps": 250, "loss": 0.0576, "learning_rate": 1.1184039683065014e-05, "epoch": 1.04, "percentage": 52.0, "elapsed_time": "0:23:18", "remaining_time": "0:21:30"}
-{"current_steps": 140, "total_steps": 250, "loss": 0.0703, "learning_rate": 9.790575801166432e-06, "epoch": 1.12, "percentage": 56.0, "elapsed_time": "0:25:05", "remaining_time": "0:19:42"}
-{"current_steps": 150, "total_steps": 250, "loss": 0.049, "learning_rate": 8.401188123081653e-06, "epoch": 1.2, "percentage": 60.0, "elapsed_time": "0:26:52", "remaining_time": "0:17:54"}
-{"current_steps": 160, "total_steps": 250, "loss": 0.129, "learning_rate": 7.042919499559538e-06, "epoch": 1.28, "percentage": 64.0, "elapsed_time": "0:28:39", "remaining_time": "0:16:07"}
-{"current_steps": 170, "total_steps": 250, "loss": 0.0652, "learning_rate": 5.742207084349274e-06, "epoch": 1.3599999999999999, "percentage": 68.0, "elapsed_time": "0:30:26", "remaining_time": "0:14:19"}
-{"current_steps": 180, "total_steps": 250, "loss": 0.0147, "learning_rate": 4.524367765074499e-06, "epoch": 1.44, "percentage": 72.0, "elapsed_time": "0:32:13", "remaining_time": "0:12:31"}
-{"current_steps": 190, "total_steps": 250, "loss": 0.0694, "learning_rate": 3.4131053988131947e-06, "epoch": 1.52, "percentage": 76.0, "elapsed_time": "0:34:00", "remaining_time": "0:10:44"}
-{"current_steps": 200, "total_steps": 250, "loss": 0.0786, "learning_rate": 2.4300494434824373e-06, "epoch": 1.6, "percentage": 80.0, "elapsed_time": "0:35:47", "remaining_time": "0:08:56"}
-{"current_steps": 210, "total_steps": 250, "loss": 0.0301, "learning_rate": 1.5943339650431578e-06, "epoch": 1.6800000000000002, "percentage": 84.0, "elapsed_time": "0:37:33", "remaining_time": "0:07:09"}
-{"current_steps": 220, "total_steps": 250, "loss": 0.025, "learning_rate": 9.222252146709143e-07, "epoch": 1.76, "percentage": 88.0, "elapsed_time": "0:39:20", "remaining_time": "0:05:21"}
-{"current_steps": 230, "total_steps": 250, "loss": 0.0093, "learning_rate": 4.268050246793276e-07, "epoch": 1.8399999999999999, "percentage": 92.0, "elapsed_time": "0:41:06", "remaining_time": "0:03:34"}
-{"current_steps": 240, "total_steps": 250, "loss": 0.0756, "learning_rate": 1.1771618553447217e-07, "epoch": 1.92, "percentage": 96.0, "elapsed_time": "0:42:53", "remaining_time": "0:01:47"}
-{"current_steps": 250, "total_steps": 250, "loss": 0.0351, "learning_rate": 9.74759906957612e-10, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:44:40", "remaining_time": "0:00:00"}
-{"current_steps": 250, "total_steps": 250, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:44:40", "remaining_time": "0:00:00"}
+{"current_steps": 10, "total_steps": 250, "loss": 1.8804, "learning_rate": 7.2000000000000005e-06, "epoch": 0.08, "percentage": 4.0, "elapsed_time": "0:00:54", "remaining_time": "0:21:55", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 20, "total_steps": 250, "loss": 0.469, "learning_rate": 1.4400000000000001e-05, "epoch": 0.16, "percentage": 8.0, "elapsed_time": "0:01:48", "remaining_time": "0:20:42", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 30, "total_steps": 250, "loss": 0.3463, "learning_rate": 1.9991228300988586e-05, "epoch": 0.24, "percentage": 12.0, "elapsed_time": "0:02:41", "remaining_time": "0:19:45", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 40, "total_steps": 250, "loss": 0.1673, "learning_rate": 1.983571470813386e-05, "epoch": 0.32, "percentage": 16.0, "elapsed_time": "0:03:35", "remaining_time": "0:18:51", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 50, "total_steps": 250, "loss": 0.106, "learning_rate": 1.9488760116444966e-05, "epoch": 0.4, "percentage": 20.0, "elapsed_time": "0:04:29", "remaining_time": "0:17:56", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 60, "total_steps": 250, "loss": 0.1623, "learning_rate": 1.895711760239413e-05, "epoch": 0.48, "percentage": 24.0, "elapsed_time": "0:05:23", "remaining_time": "0:17:03", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 70, "total_steps": 250, "loss": 0.1585, "learning_rate": 1.8251134982782952e-05, "epoch": 0.56, "percentage": 28.0, "elapsed_time": "0:06:16", "remaining_time": "0:16:09", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 80, "total_steps": 250, "loss": 0.192, "learning_rate": 1.7384553406258842e-05, "epoch": 0.64, "percentage": 32.0, "elapsed_time": "0:07:10", "remaining_time": "0:15:15", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 90, "total_steps": 250, "loss": 0.1705, "learning_rate": 1.63742398974869e-05, "epoch": 0.72, "percentage": 36.0, "elapsed_time": "0:08:04", "remaining_time": "0:14:21", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 100, "total_steps": 250, "loss": 0.0652, "learning_rate": 1.5239859059700794e-05, "epoch": 0.8, "percentage": 40.0, "elapsed_time": "0:08:58", "remaining_time": "0:13:27", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 110, "total_steps": 250, "loss": 0.1237, "learning_rate": 1.4003490325568953e-05, "epoch": 0.88, "percentage": 44.0, "elapsed_time": "0:09:52", "remaining_time": "0:12:33", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 120, "total_steps": 250, "loss": 0.1669, "learning_rate": 1.2689198206152657e-05, "epoch": 0.96, "percentage": 48.0, "elapsed_time": "0:10:46", "remaining_time": "0:11:39", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 130, "total_steps": 250, "loss": 0.0606, "learning_rate": 1.1322563902571227e-05, "epoch": 1.04, "percentage": 52.0, "elapsed_time": "0:11:39", "remaining_time": "0:10:46", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 140, "total_steps": 250, "loss": 0.1064, "learning_rate": 9.930187397020385e-06, "epoch": 1.12, "percentage": 56.0, "elapsed_time": "0:12:33", "remaining_time": "0:09:52", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 150, "total_steps": 250, "loss": 0.0548, "learning_rate": 8.539169714375885e-06, "epoch": 1.2, "percentage": 60.0, "elapsed_time": "0:13:27", "remaining_time": "0:08:58", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 160, "total_steps": 250, "loss": 0.1326, "learning_rate": 7.176585431571235e-06, "epoch": 1.28, "percentage": 64.0, "elapsed_time": "0:14:21", "remaining_time": "0:08:04", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 170, "total_steps": 250, "loss": 0.0843, "learning_rate": 5.868955701754584e-06, "epoch": 1.3599999999999999, "percentage": 68.0, "elapsed_time": "0:15:15", "remaining_time": "0:07:10", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 180, "total_steps": 250, "loss": 0.0121, "learning_rate": 4.641732050210032e-06, "epoch": 1.44, "percentage": 72.0, "elapsed_time": "0:16:08", "remaining_time": "0:06:16", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 190, "total_steps": 250, "loss": 0.0227, "learning_rate": 3.5188009893686916e-06, "epoch": 1.52, "percentage": 76.0, "elapsed_time": "0:17:02", "remaining_time": "0:05:22", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 200, "total_steps": 250, "loss": 0.0385, "learning_rate": 2.522019095014683e-06, "epoch": 1.6, "percentage": 80.0, "elapsed_time": "0:17:56", "remaining_time": "0:04:29", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 210, "total_steps": 250, "loss": 0.0295, "learning_rate": 1.6707875928990059e-06, "epoch": 1.6800000000000002, "percentage": 84.0, "elapsed_time": "0:18:49", "remaining_time": "0:03:35", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 220, "total_steps": 250, "loss": 0.0206, "learning_rate": 9.816747359488632e-07, "epoch": 1.76, "percentage": 88.0, "elapsed_time": "0:19:43", "remaining_time": "0:02:41", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 230, "total_steps": 250, "loss": 0.0089, "learning_rate": 4.6809332207053083e-07, "epoch": 1.8399999999999999, "percentage": 92.0, "elapsed_time": "0:20:37", "remaining_time": "0:01:47", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 240, "total_steps": 250, "loss": 0.0576, "learning_rate": 1.400396292949513e-07, "epoch": 1.92, "percentage": 96.0, "elapsed_time": "0:21:31", "remaining_time": "0:00:53", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 250, "total_steps": 250, "loss": 0.0308, "learning_rate": 3.898849596456477e-09, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:22:24", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 250, "total_steps": 250, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:22:24", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cefaef9b036ddb22f2bd90050e87d1acb3561f5cd20a2935a7cc9f193f551ece
+oid sha256:795e6c2b46e4ebf1af58f4737bfa7b63db4e743eeff78a7e61518c8b69fc9f50
 size 5304