Commit
•
c34440f
1
Parent(s):
dcbb05d
Update README.md
Browse files
README.md
CHANGED
|
@@ -41,6 +41,12 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
|
|
| 41 |
|
| 42 |
model_id = "Gemma2_SQLGEN"
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 45 |
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={"":0})
|
| 46 |
tokenizer.padding_side = 'right'
|
|
|
|
| 41 |
|
| 42 |
model_id = "Gemma2_SQLGEN"
|
| 43 |
|
| 44 |
+
bnb_config = BitsAndBytesConfig(
|
| 45 |
+
load_in_4bit=True,
|
| 46 |
+
bnb_4bit_quant_type="nf4",
|
| 47 |
+
bnb_4bit_compute_dtype=torch.bfloat16
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 51 |
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={"":0})
|
| 52 |
tokenizer.padding_side = 'right'
|