Add dutch partisan news dataset

.gitattributes

@@ -33,3 +33,18 @@ adapted-retriever/sentence_bert_config.json filter=lfs diff=lfs merge=lfs -text
 adapted-retriever/special_tokens_map.json filter=lfs diff=lfs merge=lfs -text
 adapted-retriever/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 adapted-retriever/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+documentstore_german-election-idx_adapted.pkl filter=lfs diff=lfs merge=lfs -text
+dutch-article-idx_adapted.pkl filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever filter=lfs diff=lfs merge=lfs -text
+dutch-article-idx.pkl filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/1_Pooling filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/README.md filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/config.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/config_sentence_transformers.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/modules.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/sentence_bert_config.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/special_tokens_map.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+dutch-article-retriever/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -3,7 +3,8 @@
 Here's our first attempt at using data to create a table:
 """
 import streamlit as st
-from retriever import do_search
+from retriever import do_search, dutch_datset_name, german_datset_name
+
 
 def local_css(file_name):
     with open(file_name) as f:
@@ -16,7 +17,7 @@ def render_retrieved_content(content, score):
     if score is not None:
         score = round(score, 3)
         print_score = f'<b> Similarity Score: {score}</b>'
-    return f'<blockquote>{content} </blockquote> {print_score}'
+    return f'<blockquote> {content} </blockquote> {print_score}'
 
 local_css('style.css')
 st.header('🧐  Where my docs at?')
@@ -31,12 +32,12 @@ st.markdown('✨ Imagine you have a bunch of text documents and looking for one
 with st.form('search-input'):
     option = st.selectbox(
         'Choose a dataset',
-        ('CDU election program 2021', 'Partisan news 2019 (dutch)'))
+        (german_datset_name, dutch_datset_name))
     search = st.text_input('Enter your search query')
     button = st.form_submit_button('Search')
 
 if search:
-    result = do_search(search)
+    result = do_search(search, option)
 
     st.markdown('### 🔎  Term Frequency–Inverse Document Frequency (TF-IDF)')
     st.markdown('Is a statistical approach that calculates how relevant a word is to a document '

dutch-article-idx.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b1edcffb6ca9c5409af117770d97415a119bcb02fc5c3ac338f82dadacdb51
+size 24987947

dutch-article-idx_adapted.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed14bfd16fa49000673d7964bf90f3da854b3a17209554bc4ec6d1664f59858d
+size 25239050

dutch-article-retriever/1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

dutch-article-retriever/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:112c56ba0758ca51e45cda7f0d505af643c740abd0af7f740ec411d30708a96d
+size 3696

dutch-article-retriever/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29703b29b31e2dabfcd73e52ba0856489249af29f2c8fc5209415fccadfac0d3
+size 821

dutch-article-retriever/config_sentence_transformers.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c64b5cece00d8424b4896ea75b512b6008576088497609dfeb6bd63e6d36b8
+size 122

dutch-article-retriever/modules.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4b264b80206c830bebbdcae377e137925650a433b689343a63bdc9b3145460
+size 229

dutch-article-retriever/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b906450207e003aaf2f08d775fedfb16b8438206899eb12a93f92059069ad8a
+size 1112244081

dutch-article-retriever/sentence_bert_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8e29d6dcb61b611b7d3fdd2982c4524e6ad985959fa7194eacfb655a8d0d51
+size 53

dutch-article-retriever/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

dutch-article-retriever/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d
+size 239

dutch-article-retriever/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46afe88da5fd71bdbab5cfab5e84c1adce59c246ea5f9341bbecef061891d0a7
+size 17082913

dutch-article-retriever/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c84cba673d65cd6fabcaf0340ae8e57b34306e01862132f4b476936917727dea
+size 483

retriever.py

@@ -1,11 +1,12 @@
 from haystack.document_stores import InMemoryDocumentStore
-from haystack.utils import convert_files_to_docs
+
 from haystack.nodes.retriever import TfidfRetriever
 from haystack.pipelines import DocumentSearchPipeline, ExtractiveQAPipeline
 from haystack.nodes.retriever import EmbeddingRetriever
-from haystack.nodes import FARMReader
 import pickle
 from pprint import pprint
+dutch_datset_name = 'Partisan news 2019 (dutch)'
+german_datset_name = 'CDU election program 2021'
 
 class ExportableInMemoryDocumentStore(InMemoryDocumentStore):
     """
@@ -22,54 +23,71 @@ class ExportableInMemoryDocumentStore(InMemoryDocumentStore):
             self.indexes = pickle.load(f)
 
 
-
-document_store = ExportableInMemoryDocumentStore(similarity='cosine')
-document_store.load_data('documentstore_german-election-idx.pkl')
-
-document_store_adapted = ExportableInMemoryDocumentStore(similarity='cosine')
-document_store_adapted.load_data('documentstore_german-election-idx.pkl')
-
-retriever = TfidfRetriever(document_store=document_store)
-base_dense_retriever = EmbeddingRetriever(
-        document_store=document_store,
-        embedding_model='sentence-transformers/paraphrase-multilingual-mpnet-base-v2',
-        model_format='sentence_transformers'
-    )
-
-fine_tuned_retriever = EmbeddingRetriever(
-        document_store=document_store_adapted,
-        embedding_model='./adapted-retriever',
-        model_format='sentence_transformers'
-)
-
-def sparse_retrieval(query):
-    """Sparse retrieval pipeline"""
-    scores = retriever._calc_scores(query)
-    p_retrieval = DocumentSearchPipeline(retriever)
-    documents = p_retrieval.run(query=query)
-    documents['documents'][0].score = list(scores[0].values())[0]
-    return documents
-
-def dense_retrieval(query, retriever='base'):
-    if retriever == 'base':
-        p_retrieval = DocumentSearchPipeline(base_dense_retriever)
-    elif retriever == 'adapted':
-        p_retrieval = DocumentSearchPipeline(fine_tuned_retriever)
+class SearchEngine():
+
+    def __init__(self, document_store_name_base, document_store_name_adpated,
+                 adapted_retriever_path):
+        self.document_store = ExportableInMemoryDocumentStore(similarity='cosine')
+        self.document_store.load_data(document_store_name_base)
+
+        self.document_store_adapted = ExportableInMemoryDocumentStore(similarity='cosine')
+        self.document_store_adapted.load_data(document_store_name_adpated)
+
+        self.retriever = TfidfRetriever(document_store=self.document_store)
+
+        self.base_dense_retriever = EmbeddingRetriever(
+            document_store=self.document_store,
+            embedding_model='sentence-transformers/paraphrase-multilingual-mpnet-base-v2',
+            model_format='sentence_transformers'
+        )
+
+        self.fine_tuned_retriever = EmbeddingRetriever(
+            document_store=self.document_store_adapted,
+            embedding_model=adapted_retriever_path,
+            model_format='sentence_transformers'
+        )
+
+    def sparse_retrieval(self, query):
+        """Sparse retrieval pipeline"""
+        scores = self.retriever._calc_scores(query)
+        p_retrieval = DocumentSearchPipeline(self.retriever)
+        documents = p_retrieval.run(query=query)
+        documents['documents'][0].score = list(scores[0].values())[0]
+        return documents
+
+    def dense_retrieval(self, query, retriever='base'):
+        if retriever == 'base':
+            p_retrieval = DocumentSearchPipeline(self.base_dense_retriever)
+            return p_retrieval.run(query=query)
+        if retriever == 'adapted':
+            p_retrieval = DocumentSearchPipeline(self.fine_tuned_retriever)
+            return p_retrieval.run(query=query)
+
+    def do_search(self, query):
+        sparse_result = self.sparse_retrieval(query)['documents'][0]
+        dense_base_result = self.dense_retrieval(query, 'base')['documents'][0]
+        dense_adapted_result = self.dense_retrieval(query, 'adapted')['documents'][0]
+        return sparse_result, dense_base_result, dense_adapted_result
+
+
+dutch_search_engine = SearchEngine('dutch-article-idx.pkl', 'dutch-article-idx_adapted.pkl',
+                                     'dutch-article-retriever')
+german_search_engine = SearchEngine('documentstore_german-election-idx.pkl',
+                                        'documentstore_german-election-idx_adapted.pkl',
+                                        'adapted-retriever')
+
+def do_search(query, dataset):
+    if dataset == german_datset_name:
+        return german_search_engine.do_search(query)
     else:
-        return None
-
-    return p_retrieval.run(query=query)
-
-
-def do_search(query):
-    sparse_result = sparse_retrieval(query)['documents'][0]
-    dense_base_result =dense_retrieval(query, retriever='base')['documents'][0]
-    dense_adapted_result = dense_retrieval(query, retriever='adapted')['documents'][0]
-    return sparse_result, dense_base_result, dense_adapted_result
+        return dutch_search_engine.do_search(query)
 
 if __name__ == '__main__':
-    query = 'Frauen'
-    result = do_search(query)
+    search_engine = SearchEngine('dutch-article-idx.pkl', 'dutch-article-idx_adapted.pkl',
+                                 'dutch-article-retriever')
+    query = 'Kindergarten'
+
+    result = search_engine.do_search(query)
     pprint(result)