Update app.py
Browse files
app.py
CHANGED
@@ -14,6 +14,7 @@ from scipy.sparse import csr_matrix, save_npz, load_npz, vstack
|
|
14 |
from qdrant_client import QdrantClient, models
|
15 |
from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
|
16 |
from fastembed_ext import FastEmbedEmbeddingsLc
|
|
|
17 |
from fastembed.sparse.splade_pp import supported_splade_models
|
18 |
from fastembed import SparseTextEmbedding, SparseEmbedding
|
19 |
from unstructured.partition.auto import partition
|
@@ -193,9 +194,13 @@ def load_models_and_documents():
|
|
193 |
n_gpu_layers=32
|
194 |
)
|
195 |
|
196 |
-
dense_model = HuggingFaceEncoder(name="mixedbread-ai/mxbai-embed-large-v1", device="cuda")
|
197 |
-
|
198 |
provider = ['CPUExecutionProvider']
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
|
200 |
sparse_model = SparseTextEmbedding(
|
201 |
'prithivida/Splade_PP_en_v2',
|
|
|
14 |
from qdrant_client import QdrantClient, models
|
15 |
from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
|
16 |
from fastembed_ext import FastEmbedEmbeddingsLc
|
17 |
+
from langchain_experimental.text_splitter import SemanticChunker
|
18 |
from fastembed.sparse.splade_pp import supported_splade_models
|
19 |
from fastembed import SparseTextEmbedding, SparseEmbedding
|
20 |
from unstructured.partition.auto import partition
|
|
|
194 |
n_gpu_layers=32
|
195 |
)
|
196 |
|
|
|
|
|
197 |
provider = ['CPUExecutionProvider']
|
198 |
+
|
199 |
+
dense_model = FastEmbedEmbeddingsLc(
|
200 |
+
model_name="mixedbread-ai/mxbai-embed-large-v1",
|
201 |
+
cache_dir=os.getenv('HF_HOME'),
|
202 |
+
providers=provider
|
203 |
+
)
|
204 |
|
205 |
sparse_model = SparseTextEmbedding(
|
206 |
'prithivida/Splade_PP_en_v2',
|