Update app.py
Browse files
app.py
CHANGED
|
@@ -46,8 +46,8 @@ model = AutoModelForCausalLM.from_pretrained(model_id, token= token,
|
|
| 46 |
torch_dtype=torch.float16,
|
| 47 |
# torch_dtype=torch.fl,
|
| 48 |
attn_implementation="flash_attention_2",
|
| 49 |
-
|
| 50 |
-
|
| 51 |
device_map='cuda',
|
| 52 |
|
| 53 |
)
|
|
|
|
| 46 |
torch_dtype=torch.float16,
|
| 47 |
# torch_dtype=torch.fl,
|
| 48 |
attn_implementation="flash_attention_2",
|
| 49 |
+
low_cpu_mem_usage=True,
|
| 50 |
+
llm_int8_enable_fp32_cpu_offload=True,
|
| 51 |
device_map='cuda',
|
| 52 |
|
| 53 |
)
|