Update app.py

app.py

@@ -77,9 +77,7 @@ from qdrant_client.models import (
     Batch,
     Filter,
     FieldCondition,
-    Datatype,
-    BinaryQuantization,
-    BinaryQuantizationConfig
+    Datatype
 )
 
 class Question(BaseModel):
@@ -185,13 +183,17 @@ def collect_files(directory, pattern):
 
     for filename in os.listdir(directory):
         if pattern in filename:
-            if filename.endswith('.msgpack'):
+            if filename.endswith('.msgpack') and (pattern == '_payload'):
+                with open(os.path.join(directory, filename), "rb") as data_file_payload:
+                    decompressed_payload = data_file_payload.read()
+                array.extend(msgpack.unpackb(decompressed_payload, raw=False))
+            elif filename.endswith('.msgpack') and (pattern == '_payload_titles'):
                 with open(os.path.join(directory, filename), "rb") as data_file_payload:
                     decompressed_payload = data_file_payload.read()
                 array.extend(msgpack.unpackb(decompressed_payload, raw=False))
             elif filename.endswith('.npz') and (pattern == '_dense'):
                 array.extend(list(np.load(os.path.join(directory, filename)).values()))
-            elif filename.endswith('.npz') and ((pattern == '_sparse') or (pattern == '_sparse_titles')):
+            elif filename.endswith('.npz') and (pattern == '_sparse'):
                 sparse_embeddings = []
                 loaded_sparse_matrix = load_npz(os.path.join(directory, filename))
                 
@@ -202,7 +204,21 @@ def collect_files(directory, pattern):
                     embedding = SparseVector(indices=indices, values=values)
                     sparse_embeddings.append(embedding)
                 array.extend(sparse_embeddings)
-            elif (filename.endswith('.npy')):
+            elif filename.endswith('.npz') and (pattern == '_sparse_titles'):
+                sparse_embeddings = []
+                loaded_sparse_matrix = load_npz(os.path.join(directory, filename))
+                
+                for i in range(loaded_sparse_matrix.shape[0]):
+                    row = loaded_sparse_matrix.getrow(i)
+                    values = row.data.tolist()
+                    indices = row.indices.tolist()
+                    embedding = SparseVector(indices=indices, values=values)
+                    sparse_embeddings.append(embedding)
+                array.extend(sparse_embeddings)
+            elif filename.endswith('.npy') and (pattern == '_ids'):
+                ids_list = np.load(os.path.join(directory, filename), allow_pickle=True).tolist()
+                array.extend(ids_list)
+            elif filename.endswith('.npy') and (pattern == '_ids_titles'):
                 ids_list = np.load(os.path.join(directory, filename), allow_pickle=True).tolist()
                 array.extend(ids_list)
                 
@@ -299,7 +315,7 @@ def load_models_and_documents():
         embeddings_path = os.path.join(os.getenv('HF_HOME'), 'embeddings')
 
         payload_path = os.path.join(embeddings_path, name + '_payload.msgpack')
-        payload_titles_path = os.path.join(embeddings_path, name + '_payload_titles.npz')
+        payload_titles_path = os.path.join(embeddings_path, name + '_payload_titles.msgpack')
         dense_path = os.path.join(embeddings_path, name + '_dense.npz')
         sparse_path = os.path.join(embeddings_path, name + '_sparse.npz')
         sparse_titles_path = os.path.join(embeddings_path, name + '_sparse_titles.npz')