Update app.py

app.py

@@ -150,17 +150,19 @@ def main(query: str, client: QdrantClient, collection_name: str, llm, dense_mode
 
     combined_docs = "\n".join(docs)
 
-    template = f"""Q: Use the following pieces of context to answer the question at the end.
-    If you don't know the answer, just say that you don't know, don't try to make up an answer.
-    Use three sentences maximum and keep the answer as concise as possible.
-    
-    {combined_docs}
-    
-    Question: {query}
-    
-    A: """
-
-    response = llm(template, stop=["Q:", "\n"], temperature=0.7)
+    response = llm.create_chat_completion(
+        messages = [
+          {"role": "system", "content": f"""Use the following pieces of context to answer the user question.
+          If you don't know the answer, just say that you don't know, don't try to make up an answer.
+          Use three sentences maximum and keep the answer as concise as possible.
+          
+          {combined_docs}"""
+          },
+          {
+              "role": "user",
+              "content": f"Question: {query}"
+          }
+      ], stop=["</s>"], temperature=0.7)
 
     text = response["choices"][0]["text"]
     print(f'TEXT: {text}')
@@ -171,7 +173,18 @@ def main(query: str, client: QdrantClient, collection_name: str, llm, dense_mode
     A :
     """
 
-    output = llm(prompt, stop=["Q:", "\n"], temperature=0.7, max_tokens=3000)['choices'][0]['text']
+    output = llm.create_chat_completion(
+        messages = [
+          {"role": "system", "content": """You are an assistant that provides summaries of texts. 
+          Your task is to create a summary that includes the main points and any important details. 
+          Present your response in bullet points."""
+          },
+          {
+              "role": "user",
+              "content": f"""Write a summary of the following text delimited by triple backquotes. Ensure the summary covers the key points of the text.
+              ```{text}```"""
+          }
+      ], stop=["</s>"], temperature=0.7, max_tokens=3000)['choices'][0]['text']
     print(f'OUTPUT: {output}')
     return output
 
@@ -193,6 +206,7 @@ def load_models_and_documents():
             repo_id="MaziyarPanahi/Llama-3-8B-Instruct-32k-v0.1-GGUF",
             filename="*Q8_0.gguf",
             verbose=False,
+            chat_format="chatml",
             n_ctx=16000,
             n_gpu_layers=32
         )