Upload 3 files

rag_chain/chain.py

@@ -128,7 +128,6 @@ def get_rag_chain(model_name: str = "gpt-4", temperature: float = 0.2) -> tuple[
     )
 
     # Compression retriever for practitioners db
-    # TODO
     practitioners_db_compression_retriever = compression_retriever_setup(
         practitioners_ensemble_retriever,
         embeddings_model="text-embedding-ada-002",
@@ -139,18 +138,18 @@ def get_rag_chain(model_name: str = "gpt-4", temperature: float = 0.2) -> tuple[
     dense_retriever_client = DenseRetrieverClient(embeddings_model=embeddings_model,
                                                   collection_name="tall_tree_db")
     tall_tree_db_dense_retriever = dense_retriever_client.get_dense_retriever(search_type="similarity",
-                                                                              k=5)
+                                                                              k=8)
     # Compression retriever for tall_tree_db
     tall_tree_db_compression_retriever = compression_retriever_setup(
         tall_tree_db_dense_retriever,
         embeddings_model="text-embedding-ada-002",
-        similarity_threshold=0.5
+        similarity_threshold=0.74
     )
 
     # Set conversation history window memory. It only uses the last k interactions.
     memory = ConversationBufferWindowMemory(memory_key="history",
                                             return_messages=True,
-                                            k=5)
+                                            k=6)
 
     # Set up runnable using LCEL
     setup_and_retrieval = {"practitioners_db": itemgetter("message")

rag_chain/retrievers_setup.py

@@ -47,12 +47,14 @@ class DenseRetrieverClient:
     @cache
     def load_qdrant_collection(self):
         """Load Qdrant collection for a given embeddings model."""
-        if self.embeddings_model == "text-embedding-ada-002":
+        # TODO: Test new OpenAI text embeddings models
+        openai_text_embeddings = ["text-embedding-ada-002"]
+        if self.embeddings_model in openai_text_embeddings:
             self.qdrant_collection = self.set_qdrant_collection(
                 OpenAIEmbeddings(model=self.embeddings_model))
         else:
             raise ValueError(
-                f"Invalid embeddings model: {self.embeddings_model}. Select 'text-embedding-ada-002' from OpenAI.")
+                f"Invalid embeddings model: {self.embeddings_model}. Valid options are {', '.join(openai_text_embeddings)}.")
 
         return self.qdrant_collection
 
@@ -156,6 +158,7 @@ def compression_retriever_setup(base_retriever, embeddings_model: str = "text-em
         base_retriever: Retriever to be filtered.
         similarity_threshold (float, optional): The similarity threshold for the EmbeddingsFilter. 
             Documents with a similarity score below this threshold will be filtered out. Defaults to 0.76 (Obtained by experimenting with text-embeddings-ada-002).
+            ** Be careful with this parameter, as it can have a big impact on the results and highly depends on the embeddings model used.
 
     Returns:
         ContextualCompressionRetriever: The created ContextualCompressionRetriever.