Update README.md
Browse files
README.md
CHANGED
@@ -33,8 +33,9 @@ print(tokenizer.decode(outputs[0]))
|
|
33 |
### Training dataset
|
34 |
dataset : [kyujinpy/KOpen-platypus](https://huggingface.co/datasets/kyujinpy/KOpen-platypus)
|
35 |
|
36 |
-
### lora_config and
|
37 |
|
|
|
38 |
bnd_config = BitsAndBytesConfig(
|
39 |
load_in_4bit = True,
|
40 |
bnb_4bit_use_double_quant = True,
|
@@ -46,8 +47,9 @@ lora_config = LoraConfig(
|
|
46 |
r = 16,
|
47 |
lora_alpha = 16,
|
48 |
lora_dropout = 0.05,
|
49 |
-
target_modules = ['
|
50 |
)
|
|
|
51 |
|
52 |
### Hardware
|
53 |
RTX 3090 Ti 24GB x 1
|
|
|
33 |
### Training dataset
|
34 |
dataset : [kyujinpy/KOpen-platypus](https://huggingface.co/datasets/kyujinpy/KOpen-platypus)
|
35 |
|
36 |
+
### lora_config and bnb_config in Training
|
37 |
|
38 |
+
```python
|
39 |
bnd_config = BitsAndBytesConfig(
|
40 |
load_in_4bit = True,
|
41 |
bnb_4bit_use_double_quant = True,
|
|
|
47 |
r = 16,
|
48 |
lora_alpha = 16,
|
49 |
lora_dropout = 0.05,
|
50 |
+
target_modules = ['gate_proj', 'up_proj', 'down_proj']
|
51 |
)
|
52 |
+
```
|
53 |
|
54 |
### Hardware
|
55 |
RTX 3090 Ti 24GB x 1
|