Duplicate from Blackroot/Llama3-RP-Lora

Co-authored-by: Coffee Vampire <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,40 @@
+---
+license: unlicense
+---
+## Overview
+
+This is a testing LORA for Llama-3 8B or Llama-3 8B Instruct. The goal for this model was to bring back some of the expressive prose and writing style of the base model, as well as shift the rather dry style of the 8B instruct.
+
+## Data Processing
+
+### Raw Data to Custom Data
+
+- Started with ~40GB of raw data
+- Aggressively selected for writing style
+- Cleaned multiple times both automatically and by hand
+- Final dataset size: 78.4MB
+- No synthetic data present in the used dataset
+
+### Data Cleaning Code
+
+Once the training is verified to be beneficial (and thus the cleaning was likely to be correct), the data cleaning code will be released.
+
+## Training Procedure
+
+### Training Framework
+
+Training was done QLORA style via [Axolotl](https://github.com/OpenAccess-AI-Collective/axolotl). The full training script along with the data processing scripts will be released similarly once the procedure is verified to benefit the model in a useful way.
+
+### Training Parameters
+
+- Base Model: Llama 3 8B (Non instruct)
+- r: 4
+- alpha: 8
+- dropout: 0
+- warmup: 45 steps
+- epochs: 2
+- lr: constant with warmup
+- optimizer: adamw (torch fused)
+- weight decay: 0.1
+- adam_b1: 0.9
+- adam_b2: 0.999

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "/home/blackroot/Desktop/unsloth/fp16/Meta-Llama-3-8B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 8,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 4,
+  "rank_pattern": {},
+  "revision": "unsloth",
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc9123b4951fa2d13868906842e25c1a1717a46563bdd9fd05db0b71c2cca9d4
+size 42002136