Update README.md
Browse files
README.md
CHANGED
|
@@ -33,5 +33,18 @@ print(tokenizer.decode(outputs[0]))
|
|
| 33 |
### Training dataset
|
| 34 |
dataset : [kyujinpy/KOpen-platypus](https://huggingface.co/datasets/kyujinpy/KOpen-platypus)
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
### Hardware
|
| 37 |
RTX 3090 Ti 24GB x 1
|
|
|
|
| 33 |
### Training dataset
|
| 34 |
dataset : [kyujinpy/KOpen-platypus](https://huggingface.co/datasets/kyujinpy/KOpen-platypus)
|
| 35 |
|
| 36 |
+
### lora_config and bnb_config in Training
|
| 37 |
+
|
| 38 |
+
bnd_config = BitsAndBytesConfig(
|
| 39 |
+
load_in_8bit = True
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
lora_config = LoraConfig(
|
| 43 |
+
r = 16,
|
| 44 |
+
lora_alpha = 16,
|
| 45 |
+
lora_dropout = 0.05,
|
| 46 |
+
target_modules = ['down_proj', 'up_proj', 'gate_proj']
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
### Hardware
|
| 50 |
RTX 3090 Ti 24GB x 1
|