Model save

README.md

@@ -2,9 +2,8 @@
 license: apache-2.0
 library_name: peft
 tags:
-- llama-factory
-- lora
 - unsloth
+- llama-factory
 - generated_from_trainer
 base_model: unsloth/llama-3-8b-Instruct-bnb-4bit
 model-index:
@@ -17,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # llama-3-8b-finetuned
 
-This model is a fine-tuned version of [unsloth/llama-3-8b-Instruct-bnb-4bit](https://huggingface.co/unsloth/llama-3-8b-Instruct-bnb-4bit) on the formatted_data dataset.
+This model is a fine-tuned version of [unsloth/llama-3-8b-Instruct-bnb-4bit](https://huggingface.co/unsloth/llama-3-8b-Instruct-bnb-4bit) on an unknown dataset.
 
 ## Model description
 
@@ -45,7 +44,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 1.0
+- num_epochs: 2.0
 - mixed_precision_training: Native AMP
 
 ### Training results

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": "unsloth",
   "target_modules": [
+    "gate_proj",
     "o_proj",
-    "up_proj",
-    "q_proj",
     "down_proj",
-    "v_proj",
     "k_proj",
-    "gate_proj"
+    "up_proj",
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:274ecca0c1ecb79ba01d0ce4b4e8f63918fe12f7b715537f3f0566c100c1f57b
+oid sha256:6c65529b4d6a3ba2634f2ec48033308cf00df957936762c26d39a2a1a5e6e0b8
 size 83945296

runs/Jun04_15-21-50_7ea7dc35b048/events.out.tfevents.1717514598.7ea7dc35b048.2109.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a707f07ea7c21866d17cbdaaf6f6df21855d0696cfadd184a34493a3d4c4136
+size 11050

trainer_log.jsonl

@@ -1,13 +1,26 @@
-{"current_steps": 10, "total_steps": 125, "loss": 1.5694, "learning_rate": 1.3846153846153847e-05, "epoch": 0.08, "percentage": 8.0, "elapsed_time": "0:01:14", "remaining_time": "0:14:13"}
-{"current_steps": 20, "total_steps": 125, "loss": 0.3644, "learning_rate": 1.985871018518236e-05, "epoch": 0.16, "percentage": 16.0, "elapsed_time": "0:02:15", "remaining_time": "0:11:52"}
-{"current_steps": 30, "total_steps": 125, "loss": 0.374, "learning_rate": 1.912783265061319e-05, "epoch": 0.24, "percentage": 24.0, "elapsed_time": "0:03:14", "remaining_time": "0:10:15"}
-{"current_steps": 40, "total_steps": 125, "loss": 0.1797, "learning_rate": 1.7640373758216075e-05, "epoch": 0.32, "percentage": 32.0, "elapsed_time": "0:04:14", "remaining_time": "0:09:00"}
-{"current_steps": 50, "total_steps": 125, "loss": 0.0685, "learning_rate": 1.5555702330196024e-05, "epoch": 0.4, "percentage": 40.0, "elapsed_time": "0:05:13", "remaining_time": "0:07:49"}
-{"current_steps": 60, "total_steps": 125, "loss": 0.1244, "learning_rate": 1.3036767451096148e-05, "epoch": 0.48, "percentage": 48.0, "elapsed_time": "0:06:11", "remaining_time": "0:06:42"}
-{"current_steps": 70, "total_steps": 125, "loss": 0.1301, "learning_rate": 1.028046256275869e-05, "epoch": 0.56, "percentage": 56.0, "elapsed_time": "0:07:10", "remaining_time": "0:05:38"}
-{"current_steps": 80, "total_steps": 125, "loss": 0.2033, "learning_rate": 7.50223521832773e-06, "epoch": 0.64, "percentage": 64.0, "elapsed_time": "0:08:10", "remaining_time": "0:04:35"}
-{"current_steps": 90, "total_steps": 125, "loss": 0.1496, "learning_rate": 4.919246547534709e-06, "epoch": 0.72, "percentage": 72.0, "elapsed_time": "0:09:08", "remaining_time": "0:03:33"}
-{"current_steps": 100, "total_steps": 125, "loss": 0.0447, "learning_rate": 2.7333967796597317e-06, "epoch": 0.8, "percentage": 80.0, "elapsed_time": "0:10:07", "remaining_time": "0:02:31"}
-{"current_steps": 110, "total_steps": 125, "loss": 0.1065, "learning_rate": 1.1155436402112785e-06, "epoch": 0.88, "percentage": 88.0, "elapsed_time": "0:11:05", "remaining_time": "0:01:30"}
-{"current_steps": 120, "total_steps": 125, "loss": 0.1084, "learning_rate": 1.921471959676957e-07, "epoch": 0.96, "percentage": 96.0, "elapsed_time": "0:12:02", "remaining_time": "0:00:30"}
-{"current_steps": 125, "total_steps": 125, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:12:31", "remaining_time": "0:00:00"}
+{"current_steps": 10, "total_steps": 250, "loss": 1.585, "learning_rate": 7.2000000000000005e-06, "epoch": 0.08, "percentage": 4.0, "elapsed_time": "0:01:51", "remaining_time": "0:44:47"}
+{"current_steps": 20, "total_steps": 250, "loss": 0.268, "learning_rate": 1.5200000000000002e-05, "epoch": 0.16, "percentage": 8.0, "elapsed_time": "0:03:39", "remaining_time": "0:42:00"}
+{"current_steps": 30, "total_steps": 250, "loss": 0.2895, "learning_rate": 1.9984407641819812e-05, "epoch": 0.24, "percentage": 12.0, "elapsed_time": "0:05:26", "remaining_time": "0:39:54"}
+{"current_steps": 40, "total_steps": 250, "loss": 0.1859, "learning_rate": 1.9809551553491918e-05, "epoch": 0.32, "percentage": 16.0, "elapsed_time": "0:07:13", "remaining_time": "0:37:57"}
+{"current_steps": 50, "total_steps": 250, "loss": 0.0984, "learning_rate": 1.944376370237481e-05, "epoch": 0.4, "percentage": 20.0, "elapsed_time": "0:09:01", "remaining_time": "0:36:04"}
+{"current_steps": 60, "total_steps": 250, "loss": 0.0668, "learning_rate": 1.889416373291298e-05, "epoch": 0.48, "percentage": 24.0, "elapsed_time": "0:10:47", "remaining_time": "0:34:11"}
+{"current_steps": 70, "total_steps": 250, "loss": 0.1228, "learning_rate": 1.8171448983351284e-05, "epoch": 0.56, "percentage": 28.0, "elapsed_time": "0:12:34", "remaining_time": "0:32:20"}
+{"current_steps": 80, "total_steps": 250, "loss": 0.193, "learning_rate": 1.7289686274214116e-05, "epoch": 0.64, "percentage": 32.0, "elapsed_time": "0:14:21", "remaining_time": "0:30:31"}
+{"current_steps": 90, "total_steps": 250, "loss": 0.1573, "learning_rate": 1.6266038113644605e-05, "epoch": 0.72, "percentage": 36.0, "elapsed_time": "0:16:09", "remaining_time": "0:28:42"}
+{"current_steps": 100, "total_steps": 250, "loss": 0.0905, "learning_rate": 1.5120428648705716e-05, "epoch": 0.8, "percentage": 40.0, "elapsed_time": "0:17:56", "remaining_time": "0:26:54"}
+{"current_steps": 110, "total_steps": 250, "loss": 0.1169, "learning_rate": 1.3875155864521031e-05, "epoch": 0.88, "percentage": 44.0, "elapsed_time": "0:19:43", "remaining_time": "0:25:06"}
+{"current_steps": 120, "total_steps": 250, "loss": 0.125, "learning_rate": 1.2554457579357906e-05, "epoch": 0.96, "percentage": 48.0, "elapsed_time": "0:21:30", "remaining_time": "0:23:18"}
+{"current_steps": 130, "total_steps": 250, "loss": 0.0576, "learning_rate": 1.1184039683065014e-05, "epoch": 1.04, "percentage": 52.0, "elapsed_time": "0:23:18", "remaining_time": "0:21:30"}
+{"current_steps": 140, "total_steps": 250, "loss": 0.0703, "learning_rate": 9.790575801166432e-06, "epoch": 1.12, "percentage": 56.0, "elapsed_time": "0:25:05", "remaining_time": "0:19:42"}
+{"current_steps": 150, "total_steps": 250, "loss": 0.049, "learning_rate": 8.401188123081653e-06, "epoch": 1.2, "percentage": 60.0, "elapsed_time": "0:26:52", "remaining_time": "0:17:54"}
+{"current_steps": 160, "total_steps": 250, "loss": 0.129, "learning_rate": 7.042919499559538e-06, "epoch": 1.28, "percentage": 64.0, "elapsed_time": "0:28:39", "remaining_time": "0:16:07"}
+{"current_steps": 170, "total_steps": 250, "loss": 0.0652, "learning_rate": 5.742207084349274e-06, "epoch": 1.3599999999999999, "percentage": 68.0, "elapsed_time": "0:30:26", "remaining_time": "0:14:19"}
+{"current_steps": 180, "total_steps": 250, "loss": 0.0147, "learning_rate": 4.524367765074499e-06, "epoch": 1.44, "percentage": 72.0, "elapsed_time": "0:32:13", "remaining_time": "0:12:31"}
+{"current_steps": 190, "total_steps": 250, "loss": 0.0694, "learning_rate": 3.4131053988131947e-06, "epoch": 1.52, "percentage": 76.0, "elapsed_time": "0:34:00", "remaining_time": "0:10:44"}
+{"current_steps": 200, "total_steps": 250, "loss": 0.0786, "learning_rate": 2.4300494434824373e-06, "epoch": 1.6, "percentage": 80.0, "elapsed_time": "0:35:47", "remaining_time": "0:08:56"}
+{"current_steps": 210, "total_steps": 250, "loss": 0.0301, "learning_rate": 1.5943339650431578e-06, "epoch": 1.6800000000000002, "percentage": 84.0, "elapsed_time": "0:37:33", "remaining_time": "0:07:09"}
+{"current_steps": 220, "total_steps": 250, "loss": 0.025, "learning_rate": 9.222252146709143e-07, "epoch": 1.76, "percentage": 88.0, "elapsed_time": "0:39:20", "remaining_time": "0:05:21"}
+{"current_steps": 230, "total_steps": 250, "loss": 0.0093, "learning_rate": 4.268050246793276e-07, "epoch": 1.8399999999999999, "percentage": 92.0, "elapsed_time": "0:41:06", "remaining_time": "0:03:34"}
+{"current_steps": 240, "total_steps": 250, "loss": 0.0756, "learning_rate": 1.1771618553447217e-07, "epoch": 1.92, "percentage": 96.0, "elapsed_time": "0:42:53", "remaining_time": "0:01:47"}
+{"current_steps": 250, "total_steps": 250, "loss": 0.0351, "learning_rate": 9.74759906957612e-10, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:44:40", "remaining_time": "0:00:00"}
+{"current_steps": 250, "total_steps": 250, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:44:40", "remaining_time": "0:00:00"}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:458f4f236202372b0f1f2aea8bc620ca6c31ffcbc03c13fdbd44790a89044910
+oid sha256:cefaef9b036ddb22f2bd90050e87d1acb3561f5cd20a2935a7cc9f193f551ece
 size 5304