Update app.py
Browse files
app.py
CHANGED
@@ -9,7 +9,7 @@ examples = [["How are you?"]]
|
|
9 |
model_id="clibrain/Llama-2-13b-ft-instruct-es-gptq-4bit"
|
10 |
config = AutoConfig.from_pretrained(model_id)
|
11 |
#config.quantization_config["use_exllama"] = True
|
12 |
-
config.quantization_config["disable_exllama"] =
|
13 |
config.quantization_config["exllama_config"] = {"version":2}
|
14 |
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
15 |
print("********************")
|
|
|
9 |
model_id="clibrain/Llama-2-13b-ft-instruct-es-gptq-4bit"
|
10 |
config = AutoConfig.from_pretrained(model_id)
|
11 |
#config.quantization_config["use_exllama"] = True
|
12 |
+
config.quantization_config["disable_exllama"] = False
|
13 |
config.quantization_config["exllama_config"] = {"version":2}
|
14 |
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
15 |
print("********************")
|