Updated code

Updated Chunk split size, Updated the logic to load and split the data, Added MultiQueryRetriever, Added thumbs up and thumbs down emoji in UI.

Database/PDF_HTML_CHROMA_DB/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbc93c81c0b90157248b40367d521888edbc2ddc4a5c989b7b27cc7e8884103a
-size 224436224
+oid sha256:2e31d552a7a4981d60910ac3e293b5d53d0ba9503a95933ca21ab3a20b64ebc8
+size 330657792

Database/PDF_HTML_CHROMA_DB/{f216dc0a-1fd0-4ec6-9374-c6b509f4406e → e9e678e2-95d4-4b02-8a1e-7c3ca59754d7}/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:311e71d48e2a495b503b772c206901c356178f3fcba063393d82b9befcb487aa
-size 43576000
+oid sha256:8dbd22f72c4c63507f549d3fe1d8350c50ba0bc9d64cc20f1d136119fb9a892e
+size 85476000

Database/PDF_HTML_CHROMA_DB/{f216dc0a-1fd0-4ec6-9374-c6b509f4406e → e9e678e2-95d4-4b02-8a1e-7c3ca59754d7}/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74789aa8d0d0a7b1d2b34adfb468c6a8c3c33257b650f7bc3071ba8960f2477c
+oid sha256:3f2080adbc1c9cd3e086e238928ed1f139b21a0ebad87348b410770e6a45b37e
 size 100

Database/PDF_HTML_CHROMA_DB/{f216dc0a-1fd0-4ec6-9374-c6b509f4406e → e9e678e2-95d4-4b02-8a1e-7c3ca59754d7}/index_metadata.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad3698c433298994d9ff4d9ff2e3a18b7a912d345e8d2eca149822335fd45aac
-size 1506322
+oid sha256:a37d72452ff59cb80ed779d0ff9ed91f9d6fe7c12adf909845168311e578c06b
+size 2956679

Database/PDF_HTML_CHROMA_DB/{f216dc0a-1fd0-4ec6-9374-c6b509f4406e → e9e678e2-95d4-4b02-8a1e-7c3ca59754d7}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4a6372ec6043b713ff14a34d8290130b954aef6980807d08be18134073cac25
-size 104000
+oid sha256:a50b6a6ebd0528902d0cbaa4d5d1c60af3a3fdc95a0738162eef134668c4d735
+size 204000

Database/PDF_HTML_CHROMA_DB/{f216dc0a-1fd0-4ec6-9374-c6b509f4406e → e9e678e2-95d4-4b02-8a1e-7c3ca59754d7}/link_lists.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:056d8a348024885261ef2b62f90bd5676434d76b5c10d3cc2ec96c0ce745b668
-size 219464
+oid sha256:8b00b245cd31e8691cd94191f3afefc59417c252bdabc5ec443aa58cf84328d4
+size 426496

Database/text_chunks_html_pdf.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb9577bdbf5534b25597d36851f6b06f37eee2dfe6b09e2a51057bbbe9da301f
-size 19476054
+oid sha256:8248c4c27db5e0950be6c4bf560164990dbd348cddb2b419d6f6764011a5a605
+size 22550517

OPM_Files/OPM_Retirement_backup-20230902T130906Z-001.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:51bf0b1f9298ed989624d7f19d7f59e12fcb89e2ba087a2a0ae91204728523b4
-size 168746379

app.py

@@ -238,6 +238,7 @@ if st.session_state["vector_db"] and st.session_state["llm"]:
                             st.write("---")  # Add a separator between entries
                     message = {"role": "assistant", "content": full_response, "Source":merged_source_doc}
                     st.session_state.messages.append(message)
+                    st.markdown("👍  👎  Create Ticket")
                 # else:
                     # with st.expander("source"):
                     #     message = {"role": "assistant", "content": full_response, "Source":""}

utils.py

@@ -16,7 +16,7 @@ from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.document_loaders import PyPDFLoader, DirectoryLoader
-from langchain.memory import ConversationBufferMemory, ConversationBufferWindowMemory
+from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
 from langchain.prompts.prompt import PromptTemplate
 from langchain.vectorstores import Chroma
@@ -33,6 +33,8 @@ from langchain.agents import load_tools
 from langchain.chat_models import ChatOpenAI
 from langchain.retrievers.multi_query import MultiQueryRetriever
 from langchain.chains import RetrievalQA
+import logging
+
 
 load_dotenv()
 
@@ -254,11 +256,14 @@ def load_text_chunks(text_chunks_pkl_dir):
 def load_ensemble_retriver(text_chunks, embeddings, chroma_vectorstore):
     """Load ensemble retiriever with BM25 and Chroma as individual retrievers"""
     bm25_retriever = BM25Retriever.from_documents(text_chunks)
-    bm25_retriever.k = 1
-    chroma_retriever = chroma_vectorstore.as_retriever(search_kwargs={"k": 1})  
+    bm25_retriever.k = 2
+    chroma_retriever = chroma_vectorstore.as_retriever(search_kwargs={"k": 3})  
     ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, chroma_retriever], weights=[0.3, 0.7])
-    # retriever_from_llm = MultiQueryRetriever.from_llm(retriever=ensemble_retriever, llm=ChatOpenAI())
-    return ensemble_retriever
+    logging.basicConfig()
+    logging.getLogger('langchain.retrievers.multi_query').setLevel(logging.INFO)
+    retriever_from_llm = MultiQueryRetriever.from_llm(retriever=ensemble_retriever,
+                                                              llm=ChatOpenAI(temperature=0))
+    return retriever_from_llm
 
 
 def load_conversational_retrievel_chain(retriever, llm):
@@ -310,7 +315,7 @@ def load_conversational_retrievel_chain(retriever, llm):
     Helpful Answer:"""
 
     prompt = PromptTemplate(input_variables=["history", "context", "question"], template=template)
-    memory = ConversationBufferWindowMemory(input_key="question", memory_key="history", k=1)
+    memory = ConversationBufferMemory(input_key="question", memory_key="history")
 
     qa = RetrievalQA.from_chain_type(
         llm=llm,