devve1 commited on
Commit
bd29a58
1 Parent(s): 2400c5a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +7 -2
app.py CHANGED
@@ -14,6 +14,7 @@ from scipy.sparse import csr_matrix, save_npz, load_npz, vstack
14
  from qdrant_client import QdrantClient, models
15
  from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
16
  from fastembed_ext import FastEmbedEmbeddingsLc
 
17
  from fastembed.sparse.splade_pp import supported_splade_models
18
  from fastembed import SparseTextEmbedding, SparseEmbedding
19
  from unstructured.partition.auto import partition
@@ -193,9 +194,13 @@ def load_models_and_documents():
193
  n_gpu_layers=32
194
  )
195
 
196
- dense_model = HuggingFaceEncoder(name="mixedbread-ai/mxbai-embed-large-v1", device="cuda")
197
-
198
  provider = ['CPUExecutionProvider']
 
 
 
 
 
 
199
 
200
  sparse_model = SparseTextEmbedding(
201
  'prithivida/Splade_PP_en_v2',
 
14
  from qdrant_client import QdrantClient, models
15
  from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
16
  from fastembed_ext import FastEmbedEmbeddingsLc
17
+ from langchain_experimental.text_splitter import SemanticChunker
18
  from fastembed.sparse.splade_pp import supported_splade_models
19
  from fastembed import SparseTextEmbedding, SparseEmbedding
20
  from unstructured.partition.auto import partition
 
194
  n_gpu_layers=32
195
  )
196
 
 
 
197
  provider = ['CPUExecutionProvider']
198
+
199
+ dense_model = FastEmbedEmbeddingsLc(
200
+ model_name="mixedbread-ai/mxbai-embed-large-v1",
201
+ cache_dir=os.getenv('HF_HOME'),
202
+ providers=provider
203
+ )
204
 
205
  sparse_model = SparseTextEmbedding(
206
  'prithivida/Splade_PP_en_v2',