Update app.py

app.py

@@ -178,15 +178,15 @@ def load_models_and_documents():
         tokenizer = AutoTokenizer.from_pretrained(model_path)
         
         llm = vllm.LLM(
-            model_path, 
+            model_path,
             tensor_parallel_size=1,
             max_model_len=12288,
             trust_remote_code=True,
             enforce_eager=True,
-            quantization='awq',
+            quantization="bitsandbytes",
             gpu_memory_utilization=0.9,
-            dtype='auto'
-            #load_format='npcache'
+            dtype='auto',
+            load_format="bitsandbytes"
         )
         model = models.VLLM(llm)