Update app.py

app.py

@@ -13,8 +13,7 @@ from llama_cpp import Llama
 from scipy.sparse import csr_matrix, save_npz, load_npz, vstack
 from qdrant_client import QdrantClient, models
 from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
-from statistical_chunker import StatisticalChunker
-from semantic_router.encoders.huggingface import HuggingFaceEncoder
+from fastembed_ext import FastEmbedEmbeddingsLc
 from fastembed.sparse.splade_pp import supported_splade_models
 from fastembed import SparseTextEmbedding, SparseEmbedding
 from unstructured.partition.auto import partition
@@ -103,7 +102,7 @@ def transform_query(query: str) -> str:
     return f'Represent this sentence for searching relevant passages: {query}'
 
 def main(query: str, client: QdrantClient, collection_name: str, llm, dense_model, sparse_model):
-    dense_query = list(dense_model(transform_query(query)).cpu().numpy())
+    dense_query = list(dense_model.embed_query(query,32)
     sparse_query = list(sparse_model.embed(query, 32))
 
     search_results = search(
@@ -337,11 +336,6 @@ def chunk_documents(texts, metadatas, dense_model, sparse_model):
         breakpoint_threshold_type='standard_deviation'
     )
 
-
-
-
-
-
     _metadatas = metadatas or [{}] * len(texts)
     documents = []
     metadatas_docs = []
@@ -359,7 +353,7 @@ def chunk_documents(texts, metadatas, dense_model, sparse_model):
     joblib.Parallel(n_jobs=joblib.cpu_count(), verbose=1, require='sharedmem')(
         joblib.delayed(create_document)(text, i, _metadatas) for i, text in enumerate(texts))
     
-    dense_embeddings = dense_model.embed(documents).cpu().numpy()
+    dense_embeddings = dense_model.embed_documents(documents, 32)
     sparse_embeddings = list(sparse_model.embed(documents, 32))
     
     return documents, metadatas_docs, dense_embeddings, sparse_embeddings