Spaces:
Starting
on
T4
Starting
on
T4
Update app.py
Browse files
app.py
CHANGED
@@ -8,7 +8,7 @@ import joblib
|
|
8 |
import validators
|
9 |
import numpy as np
|
10 |
import streamlit as st
|
11 |
-
from typing import List
|
12 |
from numpy import ndarray
|
13 |
from llama_cpp import Llama
|
14 |
from statistical_chunker import StatisticalChunker
|
@@ -104,7 +104,7 @@ def transform_query(query: str) -> str:
|
|
104 |
"""
|
105 |
return f'Represent this sentence for searching relevant passages: {query}'
|
106 |
|
107 |
-
def main(query: str, client: QdrantClient, collection_name: str, llm, dense_model, sparse_model):
|
108 |
dense_query = list(dense_model(query,32))
|
109 |
sparse_query = list(sparse_model.embed(query, 32))
|
110 |
|
@@ -327,7 +327,7 @@ def load_models_and_documents():
|
|
327 |
|
328 |
return client, collection_name, llm, dense_model, sparse_model
|
329 |
|
330 |
-
def chunk_documents(texts, metadatas, dense_model, sparse_model):
|
331 |
import time
|
332 |
text_splitter = StatisticalChunker(
|
333 |
dense_model
|
|
|
8 |
import validators
|
9 |
import numpy as np
|
10 |
import streamlit as st
|
11 |
+
from typing import List
|
12 |
from numpy import ndarray
|
13 |
from llama_cpp import Llama
|
14 |
from statistical_chunker import StatisticalChunker
|
|
|
104 |
"""
|
105 |
return f'Represent this sentence for searching relevant passages: {query}'
|
106 |
|
107 |
+
def main(query: str, client: QdrantClient, collection_name: str, llm, dense_model: FastEmbedEncoder, sparse_model: SparseTextEmbedding):
|
108 |
dense_query = list(dense_model(query,32))
|
109 |
sparse_query = list(sparse_model.embed(query, 32))
|
110 |
|
|
|
327 |
|
328 |
return client, collection_name, llm, dense_model, sparse_model
|
329 |
|
330 |
+
def chunk_documents(texts: List[str], metadatas: List[dict], dense_model: FastEmbedEncoder, sparse_model: SparseTextEmbedding):
|
331 |
import time
|
332 |
text_splitter = StatisticalChunker(
|
333 |
dense_model
|