Update app.py

app.py

@@ -131,7 +131,8 @@ def load_models_and_documents():
             chat_format="llama-3",
             n_ctx=16384,
             n_gpu_layers=32,
-            flash_attn=True
+            flash_attn=True,
+            type_k='q8_0'
         )
 
         dense_model = OptimumEncoder(