Update app.py

app.py

@@ -287,22 +287,11 @@ def chunk_documents(texts: List[str], metadatas: List[dict], dense_model: Optimu
         dense_model,
         breakpoint_threshold_type='standard_deviation'
     )
-    _metadatas = metadatas or [{}] * len(texts)
-    documents = []
-    metadatas_docs = []
-
-    def create_document(text: str, i: int, _metadatas):
-        index = -1
-        for chunk in text_splitter.split_text(text):
-            metadata = copy.deepcopy(_metadatas[i])
-            if text_splitter._add_start_index:
-                index = text.find(chunk, index + 1)
-                metadata['start_index'] = index      
-        documents.append(chunk)
-        metadatas_docs.append(metadata)
-        
-    joblib.Parallel(n_jobs=joblib.cpu_count(), verbose=1, require='sharedmem')(
-        joblib.delayed(create_document)(text, i, _metadatas) for i, text in enumerate(texts))
+    docs = text_splitter.create_documents(texts, metadatas)
+    documents, metadatas_docs = zip(*[(doc.page_content, doc.metadata) for doc in documents])
+
+    documents = list(documents)
+    metadatas_docs = list(metadatas_docs)
 
     start_dense = time.time()
     dense_embeddings = dense_model.embed_documents(documents, 32, convert_to_numpy=True)