Update app.py

app.py

@@ -11,10 +11,18 @@ from pathlib import Path
 from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
 from langdetect import detect
 CONTEXT_WINDOW = 50_000
+from transformers import BitsAndBytesConfig
 
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype="float16",
+    bnb_4bit_use_double_quant=True
+)
 llm = HuggingFaceEndpoint(
     repo_id="mistralai/Mistral-Nemo-Instruct-2407", #"mistralai/Mistral-7B-Instruct-v0.3",
     task="text-generation",
+    model_kwargs={"quantization_config": quantization_config},
     max_new_tokens=4096,
     temperature=0.5,
     do_sample=False,