Update app.py

app.py

@@ -341,18 +341,27 @@ def chunk_documents(texts, metadatas, dense_model, sparse_model):
         dense_model,
         breakpoint_threshold_type='standard_deviation'
     )
-    docs = []
-    for (text, metadata) in zip(texts, metadatas):
-        docs.append(Document(page_content=text, metadata=metadata))
+    _metadatas = metadatas or [{}] * len(texts)
+    documents = []
+    metadatas_docs = []
+
+    def create_document(text: str, i: int, _metadatas):
+        index = -1
+        for chunk in text_splitter.split_text(text):
+            metadata = copy.deepcopy(_metadatas[i])
+            if text_splitter._add_start_index:
+                index = text.find(chunk, index + 1)
+                metadata['start_index'] = index      
+        documents.append(chunk)
+        metadatas_docs.append(metadata)
         
-    chunks = [doc for doc in text_splitter.transform_documents(docs)]
-    texts_docs = [chunk.page_content for chunk in chunks]
-    metadatas_docs = [chunk.metadata for chunk in chunks]
+    joblib.Parallel(n_jobs=joblib.cpu_count(), verbose=1, require='sharedmem')(
+        joblib.delayed(create_document)(text, i, _metadatas) for i, text in enumerate(texts))
     
-    dense_embeddings = dense_model.embed_documents(texts_docs, 32)
-    sparse_embeddings = list(sparse_model.embed(texts_docs, 32))
+    dense_embeddings = dense_model.embed_documents(documents, 32)
+    sparse_embeddings = list(sparse_model.embed(documents, 32))
     
-    return texts_docs, metadatas_docs, dense_embeddings, sparse_embeddings
+    return documents, metadatas_docs, dense_embeddings, sparse_embeddings
 
 def on_change_documents_only():
     st.session_state.qa_prompt = lambda query, context: (