notebooks/indexing.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["# Indexing\n","Using [Haystack](https://github.com/deepset-ai/haystack), the following steps are performed:\n","- load and preprocess documents downloaded from Wikipedia\n","- create document store and write documents\n","- initialize retriever and generate document embeddings"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"outputs":[],"source":["! pip install farm-haystack[faiss-gpu]==1.7.0"]},{"cell_type":"markdown","metadata":{},"source":["## Load documents"]},{"cell_type":"code","execution_count":2,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:23.692554Z","iopub.status.busy":"2022-08-21T08:23:23.692208Z","iopub.status.idle":"2022-08-21T08:23:23.700721Z","shell.execute_reply":"2022-08-21T08:23:23.698130Z","shell.execute_reply.started":"2022-08-21T08:23:23.692512Z"},"trusted":true},"outputs":[],"source":["import glob, json"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:23.707774Z","iopub.status.busy":"2022-08-21T08:23:23.704107Z","iopub.status.idle":"2022-08-21T08:23:25.026910Z","shell.execute_reply":"2022-08-21T08:23:25.025990Z","shell.execute_reply.started":"2022-08-21T08:23:23.705010Z"},"trusted":true},"outputs":[],"source":["docs=[]\n","\n","for json_file in glob.glob('../input/crawl-rock/rock_wiki/*.json'):\n","    with open(json_file, 'r') as fin:\n","        doc=json.load(fin)\n","\n","    docs.append(doc)\n"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:25.030530Z","iopub.status.busy":"2022-08-21T08:23:25.029931Z","iopub.status.idle":"2022-08-21T08:23:25.039324Z","shell.execute_reply":"2022-08-21T08:23:25.037960Z","shell.execute_reply.started":"2022-08-21T08:23:25.030491Z"},"trusted":true},"outputs":[{"data":{"text/plain":["453"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["len(docs)"]},{"cell_type":"markdown","metadata":{},"source":["## Preprocess documents"]},{"cell_type":"code","execution_count":6,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:25.050479Z","iopub.status.busy":"2022-08-21T08:23:25.050099Z","iopub.status.idle":"2022-08-21T08:23:42.089083Z","shell.execute_reply":"2022-08-21T08:23:42.087929Z","shell.execute_reply.started":"2022-08-21T08:23:25.050446Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"108e8c46426f44e7be98a8ae930d81ce","version_major":2,"version_minor":0},"text/plain":["Preprocessing:   0%|          | 0/453 [00:00<?, ?docs/s]"]},"metadata":{},"output_type":"display_data"}],"source":["# preprocess documents, splitting by chunks of 2 sentences\n","\n","from haystack.nodes import PreProcessor\n","\n","processor = PreProcessor(\n","    clean_empty_lines=True,\n","    clean_whitespace=True,\n","    clean_header_footer=True,\n","    split_by=\"sentence\",\n","    split_length=2,\n","    split_respect_sentence_boundary=False,\n","    split_overlap=0,\n","    language ='en'\n",")\n","preprocessed_docs = processor.process(docs)"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:42.092031Z","iopub.status.busy":"2022-08-21T08:23:42.090654Z","iopub.status.idle":"2022-08-21T08:23:42.105757Z","shell.execute_reply":"2022-08-21T08:23:42.104500Z","shell.execute_reply.started":"2022-08-21T08:23:42.091989Z"},"trusted":true},"outputs":[{"data":{"text/plain":["50024"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["len(preprocessed_docs)"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:42.108367Z","iopub.status.busy":"2022-08-21T08:23:42.107604Z","iopub.status.idle":"2022-08-21T08:23:42.117080Z","shell.execute_reply":"2022-08-21T08:23:42.115996Z","shell.execute_reply.started":"2022-08-21T08:23:42.108271Z"},"trusted":true},"outputs":[{"data":{"text/plain":["[<Document: {'content': 'Disturbed is an American heavy metal band from Chicago, formed in 1994. The band includes vocalist David Draiman, guitarist/keyboardist Dan Donegan, bassist John Moyer, and drummer Mike Wengren.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 0}, 'embedding': None, 'id': '543d4f9f9023bfc277edf307a6aef870'}>,\n"," <Document: {'content': 'Donegan and Wengren have been involved in the band since its inception, with Moyer replacing former bassist Steve \"Fuzz\" Kmak and Draiman replacing original lead vocalist Erich Awalt. The band has released seven studio albums, five of which have consecutively debuted at number one on the Billboard 200.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 1}, 'embedding': None, 'id': 'dfb0ef877837c95b2e8b03cfe2ae2057'}>,\n"," <Document: {'content': \"Disturbed went into hiatus in October 2011, during which the band's members focused on various side projects, and returned in June 2015, releasing their first album in four years, Immortalized in August 2015. They also released two live albums, Music as a Weapon II in February 2004 and Disturbed: Live at Red Rocks in November 2016.\", 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 2}, 'embedding': None, 'id': 'e498da0cc7477f698f4a30c85dbfd95d'}>,\n"," <Document: {'content': 'With over 17 million records sold worldwide, Disturbed ranks alongside Slipknot and Godsmack as one of the most successful rock bands of the 21st century. == History ==\\n\\n=== Early years (1994–1996) ===\\nBefore David Draiman joined Disturbed, guitarist Dan Donegan, drummer Mike Wengren and bassist Steve \"Fuzz\" Kmak were in a band called Brawl with vocalist Erich Awalt.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 3}, 'embedding': None, 'id': '2b51b8f38befc2c53c65c66af6de1e05'}>,\n"," <Document: {'content': 'Before changing their name to \"Brawl\", however, Donegan mentioned in the band\\'s DVD, Decade of Disturbed, that the name was originally going to be \"Crawl\"; they switched it to \"Brawl\", due to the name already being used by another band. Awalt left the band shortly after the recording of a demo tape; the other three members advertised for a singer.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 4}, 'embedding': None, 'id': 'c9b3a4a74f8332c9d4c3d1e30cac49f7'}>,\n"," <Document: {'content': 'They posted an advertisement in the local music publication in Chicago, Illinois, called the \"Illinois Entertainer\". Draiman answered the advertisement after going to twenty other auditions that month.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 5}, 'embedding': None, 'id': '946a4a27f2f1838ec070a951dab2e1b0'}>,\n"," <Document: {'content': 'Guitarist Dan Donegan commented on Draiman: \"You know, out of all the singers that we had talked to or auditioned, he [Draiman] was the only singer who was ready to go with originals. And that impressed me, just to attempt that\".With regard to Draiman being the singer for the band, Donegan said, \"After a minute or two, he just starts banging out these melodies that were huge...I\\'m playing my guitar and I\\'m grinning from ear to ear, trying not to give it away that I like this guy, you know, because I don\\'t want to, you know...[say] \\'Yeah, we\\'ll give you a call back.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 6}, 'embedding': None, 'id': 'ac2bea954f4d19cc0e48868fb503e23e'}>,\n"," <Document: {'content': \"We'll, you know, discuss it.' But I was so psyched.\", 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 7}, 'embedding': None, 'id': '852a32a0d3ce1eefa48d6420fab35dba'}>,\n"," <Document: {'content': 'Chill up my spine. I\\'m like, \\'There is something here.\\'\"', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 8}, 'embedding': None, 'id': 'ebb584a0fee18b51cd14ceadcb4a7bb8'}>,\n"," <Document: {'content': 'As drummer Mike Wengren commented, \"We clicked right off the bat.\" Draiman then joined the band in 1996 and the band was renamed Disturbed.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 9}, 'embedding': None, 'id': '51340b7bf229e5b8d4460341f7aaa9d0'}>]"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["preprocessed_docs[:10]"]},{"cell_type":"markdown","metadata":{},"source":["## Create document store ([FAISS](https://github.com/facebookresearch/faiss)) and write documents"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:42.119585Z","iopub.status.busy":"2022-08-21T08:23:42.118544Z","iopub.status.idle":"2022-08-21T08:23:42.124669Z","shell.execute_reply":"2022-08-21T08:23:42.123597Z","shell.execute_reply.started":"2022-08-21T08:23:42.119551Z"},"trusted":true},"outputs":[],"source":["from haystack.document_stores import FAISSDocumentStore\n","from haystack.nodes import EmbeddingRetriever"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:23:42.129562Z","iopub.status.busy":"2022-08-21T08:23:42.128772Z","iopub.status.idle":"2022-08-21T08:23:42.259879Z","shell.execute_reply":"2022-08-21T08:23:42.258950Z","shell.execute_reply.started":"2022-08-21T08:23:42.129518Z"},"trusted":true},"outputs":[],"source":["# the document store settings are those compatible with Embedding Retriever\n","document_store = FAISSDocumentStore(\n","    similarity=\"dot_product\",\n","    embedding_dim=768)"]},{"cell_type":"code","execution_count":46,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:43:25.952230Z","iopub.status.busy":"2022-08-21T08:43:25.951856Z","iopub.status.idle":"2022-08-21T08:46:12.506842Z","shell.execute_reply":"2022-08-21T08:46:12.505845Z","shell.execute_reply.started":"2022-08-21T08:43:25.952198Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"dbd72ecf0d36401ba26826f7d9a42540","version_major":2,"version_minor":0},"text/plain":["Writing Documents:   0%|          | 0/50024 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["# write documents\n","document_store.write_documents(preprocessed_docs)"]},{"cell_type":"markdown","metadata":{},"source":["## Initialize retriever (Embedding Retriever) and generate document embeddings\n","We choose a Sentence Tranformer model that is suitable for asymmetric semantic search (short query and longer passages), according to [documentation](https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search)."]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:56:25.360959Z","iopub.status.busy":"2022-08-21T08:56:25.360546Z","iopub.status.idle":"2022-08-21T08:58:07.214654Z","shell.execute_reply":"2022-08-21T08:58:07.213653Z","shell.execute_reply.started":"2022-08-21T08:56:25.360926Z"},"trusted":true},"outputs":[],"source":["from haystack.nodes import EmbeddingRetriever\n","\n","retriever = EmbeddingRetriever(\n","    document_store=document_store,\n","   embedding_model=\"sentence-transformers/msmarco-distilbert-base-tas-b\",\n","   model_format=\"sentence_transformers\",\n","    embed_meta_fields=['name']\n",")\n","\n","# generate embeddings\n","document_store.update_embeddings(retriever)"]},{"cell_type":"markdown","metadata":{},"source":["## Save and export index"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import shutil\n","import glob"]},{"cell_type":"code","execution_count":73,"metadata":{"execution":{"iopub.execute_input":"2022-08-21T08:58:33.494417Z","iopub.status.busy":"2022-08-21T08:58:33.493822Z","iopub.status.idle":"2022-08-21T08:58:33.635915Z","shell.execute_reply":"2022-08-21T08:58:33.634599Z","shell.execute_reply.started":"2022-08-21T08:58:33.494382Z"},"trusted":true},"outputs":[],"source":["OUT_DIR = 'YOUR-OUT-DIR'\n","\n","document_store.save(\"my_faiss_index.faiss\")\n","for f in glob.glob('*faiss*.*')+glob.glob('faiss*.*'):\n","  shutil.copy(f, OUT_DIR)"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}