Update app.py

app.py

@@ -283,18 +283,26 @@ def load_models_and_documents():
 
 def chunk_documents(texts: List[str], metadatas: List[dict], dense_model: OptimumEncoder, sparse_model: SparseTextEmbedding):
     import time
-    text_splitter = StatisticalChunker(
-        dense_model
+    text_splitter = SemanticChunker(
+        dense_model,
+        breakpoint_threshold_type='standard_deviation'
     )
-    start = time.time()
-    chunks = text_splitter(docs=texts, metadatas=metadatas)
-    end = time.time()
-    final = end - start
-    print(f'FINAL CHUNKING TIME: {final}')
-
-    documents_and_metadatas = [(chunk.content, chunk.metadata) for sub_chunk in chunks for chunk in sub_chunk]
-    documents, metadatas_docs = [list(t) for t in zip(*documents_and_metadatas)]
-    print(f'CHUNKS : {documents}')
+    _metadatas = metadatas or [{}] * len(texts)
+    documents = []
+    metadatas_docs = []
+
+    def create_document(text: str, i: int, _metadatas):
+        index = -1
+        for chunk in text_splitter.split_text(text):
+            metadata = copy.deepcopy(_metadatas[i])
+            if text_splitter._add_start_index:
+                index = text.find(chunk, index + 1)
+                metadata['start_index'] = index      
+        documents.append(chunk)
+        metadatas_docs.append(metadata)
+        
+    joblib.Parallel(n_jobs=joblib.cpu_count(), verbose=1, require='sharedmem')(
+        joblib.delayed(create_document)(text, i, _metadatas) for i, text in enumerate(texts))
 
     start_dense = time.time()
     dense_embeddings = dense_model(documents, 32, convert_to_numpy=True)