Update app.py

app.py

@@ -18,7 +18,8 @@ from typing import Iterable
 from qdrant_client import AsyncQdrantClient, models
 from fastembed.sparse.splade_pp import supported_splade_models
 from fastembed import SparseTextEmbedding, SparseEmbedding
-from langchain_community.chat_models.ollama import ChatOllama
+from langchain_community.llms.llamacpp import LlamaCpp
+from langchain_core.callbacks import CallbackManager, StreamingStdOutCallbackHandler
 from fastembed_ext import FastEmbedEmbeddingsLc
 from langchain_community.document_loaders.wikipedia import WikipediaLoader
 from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
@@ -252,12 +253,16 @@ async def main(query: str, chunks: list[str], llm, dense_model, sparse_model):
 async def load_models_and_components():
     model_name = 'adrienbrault/nous-hermes2theta-llama3-8b:q8_0'
 
-    llm = ChatOllama(
-        model=model_name, 
-        num_ctx=8192,
-        temperature=0.8,
-        f16_kv=True,
-        num_predict=3000
+    llm = LlamaCpp(
+        model_path=model_path,
+        temperature=0.75,
+        max_tokens=3000,
+        n_ctx=8192,
+        top_p=1,
+        callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]),
+        verbose=True,
+        n_gpu_layers=0,
+        n_batch=512
     )
         
     provider = ['OpenVINOExecutionProvider']