Spaces:

minhdang
/

code

Paused

minhdang commited on Mar 19

Commit

c2bc52a

•

1 Parent(s): 8ef6bba

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -9,7 +9,9 @@ from transformers import BitsAndBytesConfig
 nf4_config = BitsAndBytesConfig(
     load_in_4bit=True,
-   bnb_4bit_use_double_quant=True,
 )
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024

 nf4_config = BitsAndBytesConfig(
     load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
 )
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024