Update app.py

app.py

@@ -28,6 +28,7 @@ from qdrant_client.models import (
     SearchRequest,
     ScoredPoint,
 )
+from llama_cpp import Llama
 
 MAP_PROMPT = """
 You will be given a single passage of a book. This section will be enclosed in triple backticks (```)
@@ -155,38 +156,28 @@ def main(query: str, client: QdrantClient, collection_name: str, llm, dense_mode
 
     combined_docs = "\n".join(docs)
 
-    template = """Use the following pieces of context to answer the question at the end.
+    template = f"""Q: Use the following pieces of context to answer the question at the end.
     If you don't know the answer, just say that you don't know, don't try to make up an answer.
     Use three sentences maximum and keep the answer as concise as possible.
-    Always say "thanks for asking!" at the end of the answer.
     
-    {context}
+    {combined_docs}
     
-    Question: {question}
+    Question: {query}
     
-    Helpful Answer:"""
-    
-    rag_prompt = PromptTemplate.from_template(template)
+    A: """
 
-    chain = {"context": combined_docs, "question": RunnablePassthrough()} | rag_prompt | llm | StrOutputParser()
-    
-    output = chain.invoke(query)
-    print('THHFGHFGBFGBFB')
+    response = llm(template, stop=["Q:", "\n"], temperature=0.7)
 
-    prompt = """Write a summary of the following text delimited by triple backquotes that includes the main points and any important details.
+    text = response["choices"][0]["text"]
+
+    prompt = f"""Q: Write a summary of the following text delimited by triple backquotes that includes the main points and any important details.
     Return your response in bullet points which covers the key points of the text.
     ```{text}```
-    BULLET POINT SUMMARY:
+    A :
     """
-    
-    summary_chain = load_summarize_chain(llm,
-                                        "stuff",
-                                        prompt=prompt
-                                        )
-    
-    summary = summary_chain.invoke(output['output_text'])
-    print('ADSDGFGVHBTFEFSDGFES')
-    return summary['output_text']
+
+    output = llm(prompt, stop=["Q:", "\n"], temperature=0.7, max_tokens)
+    return output["choices"][0]["text"]
 
 @st.cache_resource
 def load_models_and_documents():
@@ -202,17 +193,12 @@ def load_models_and_documents():
     }
     
     with st.spinner('Load models...'):
-        model_path = hf_hub_download(repo_id='MaziyarPanahi/Llama-3-8B-Instruct-32k-v0.1-GGUF',
-                                     filename='Llama-3-8B-Instruct-32k-v0.1.Q8_0.gguf'
-                                     )
-        
-        llm = LlamaCpp(
-            model_path=model_path,
-            n_ctx=32000,
-            max_tokens=2000,
-            n_gpu_layers=32,
-            n_batch=256,
-            f16_kv=True
+        llm = Llama.from_pretrained(
+            repo_id="MaziyarPanahi/Llama-3-8B-Instruct-32k-v0.1-GGUF",
+            filename="*Q8_0.gguf",
+            verbose=False,
+            n_ctx=16000,
+            n_gpu_layers=32
         )
 
         provider = ['CPUExecutionProvider']