Upload 5 files

app.py

@@ -30,9 +30,15 @@ st.title("Instructor XL Embeddings")
 
 
 st.write(
-    "The app compares the performance of the Instructor-XL Embedding Model on the text from AMD's Q1 2020 Earnings Call Transcript."
+    """The app compares the performance of different instructions using the Instructor-XL Embedding Model on the text from AMD's Q1 2020 Earnings Call Transcript. 
+    The app uses a two stage retreival process: 
+    1. BM-25 to filter the results based on keyword matching, 
+    2. Instructor-XL to perform Semantic Search."""
 )
 
+use_bm25 = st.checkbox('Use BM25 for filtering results')
+
+
 data = get_data()
 
 
@@ -86,6 +92,7 @@ with col1:
         st.number_input("Number of Results to query", 1, 15, value=5)
     )
 
+
 corpus, bm25 = get_bm25_model(data)
 
 tokenized_query = preprocess_text(query_text).split()
@@ -127,35 +134,36 @@ index_mapping = {
 }
 
 with col2:
-    with st.form("my_form"):
-        text_embedding_instruction = st.selectbox(
-            "Select instruction for Text Embedding",
-            text_embedding_instructions_choice,
-        )
+    text_embedding_instruction = st.selectbox(
+        "Select instruction for Text Embedding",
+        text_embedding_instructions_choice,
+    )
+    pinecone_index_name = index_mapping[text_embedding_instruction]
+    pinecone.init(
+        api_key=st.secrets[f"pinecone_{pinecone_index_name}"],
+        environment="asia-southeast1-gcp-free",
+    )
 
-        submitted = st.form_submit_button("Submit")
-        if submitted:
-            pinecone_index_name = index_mapping[text_embedding_instruction]
-            pinecone.init(
-                api_key=st.secrets[f"pinecone_{pinecone_index_name}"],
-                environment="asia-southeast1-gcp-free",
-            )
-
-            pinecone_index = pinecone.Index(pinecone_index_name)
-
-            matches = query_pinecone(
-                dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=indices
-            )
-            context = format_query(matches)
-            output_text = format_context(context)
-
-            st.subheader("Retrieved Text:")
-            for output in output_text:
-                output = f"""{output}"""
-                st.write(
-                    f"<ul><li><p>{output}</p></li></ul>",
-                    unsafe_allow_html=True,
-                )
+    pinecone_index = pinecone.Index(pinecone_index_name)
+    
+    if use_bm25==True:
+        matches = query_pinecone(
+            dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=indices
+        )
+    else:
+        matches = query_pinecone(
+            dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=None
+        )
+    context = format_query(matches)
+    output_text = format_context(context)
+
+    st.subheader("Retrieved Text:")
+    for output in output_text:
+        output = f"""{output}"""
+        st.write(
+            f"<ul><li><p>{output}</p></li></ul>",
+            unsafe_allow_html=True,
+        )
 
 
 file_text = retrieve_transcript()

utils/retriever.py

@@ -6,15 +6,23 @@ def query_pinecone(
     dense_vec,
     top_k,
     index,
-    indices
+    indices=None
 ):
-    xc = index.query(
-        vector=dense_vec,
-        top_k=top_k,              
-        filter={"QA_Flag": {"$eq": "Answer"},
-        "index": {"$in": indices}},
-        include_metadata=True,
-    )
+    if indices != None:
+        xc = index.query(
+            vector=dense_vec,
+            top_k=top_k,              
+            filter={"QA_Flag": {"$eq": "Answer"},
+            "index": {"$in": indices}},
+            include_metadata=True,
+        )
+    else:
+        xc = index.query(
+            vector=dense_vec,
+            top_k=top_k,              
+            filter={"QA_Flag": {"$eq": "Answer"}},
+            include_metadata=True,
+        )
     return xc["matches"]