final 1 task

Dataset/embeddingsbooks.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f8f45afeee67807066143b91d360a9612d22e24bd71857020a3e32c271292d2
-size 22172986
+oid sha256:53781aa6a45e47ac88bdc1dfa593f09d835d51c3d9eb366bc01ab89dea13559c
+size 27330999

Dataset/faiss.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:422b6dabe40fb0b12a2b62cc06bffcf61ca5294e9da279dd6c4abc56c1310881
+size 7260909

app.py

@@ -4,19 +4,30 @@ import torch
 from PIL import Image
 from io import BytesIO
 import requests
+import faiss
 
 
 from transformers import AutoTokenizer, AutoModel
 import numpy as np
+st.set_page_config(layout="wide")
 
 @st.cache_resource()
 def load_model():
     model = AutoModel.from_pretrained("cointegrated/rubert-tiny2")
-    return model
+    tokenizer = AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
+    return model , tokenizer
 
-model = load_model()
-tokenizer = AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
+model, tokenizer = load_model()
 
+@st.cache_data()
+def load_data():
+    df = pd.read_csv('Dataset/parcedbooks.csv')
+    with open('Dataset/embeddingsbooks.txt', 'r') as file:
+        embeddings_list = [list(map(float, line.split())) for line in file.readlines()]
+    index = faiss.read_index('Dataset/faiss.index')
+    return df, embeddings_list, index
+
+df, embeddings_list, index = load_data()
 
 def embed_bert_cls(text, model, tokenizer):
     t = tokenizer(text, padding=True, truncation=True, return_tensors='pt')
@@ -26,38 +37,29 @@ def embed_bert_cls(text, model, tokenizer):
     embeddings = torch.nn.functional.normalize(embeddings)
     return embeddings[0].cpu().numpy()
 
-df = pd.read_csv('Dataset/parcedbooks.csv')
-
-
 
 text = st.text_input('Введите ваше предпочтение для рекомендации')
 
 button = st.button('Отправить запрос')
-num = st.number_input('Укажите количество книг для рекомендации', step=1, value=None)
-with open('Dataset/embeddingsbooks.txt', 'r') as file:
-    embeddings_list = [list(map(float, line.split())) for line in file.readlines()]
+num = st.number_input('Укажите количество книг для рекомендации', step=1, value=1)
+
 
 if text and button:
     decode_text = embed_bert_cls(text, model, tokenizer)  # Получение вектора для введенного текста
-    cosine_similarities = []
-    for annotation in embeddings_list:
-        similarity = np.dot(decode_text, annotation) / (np.linalg.norm(decode_text) * np.linalg.norm(annotation))  # Расчет косинусного сходства
-        cosine_similarities.append(similarity)
+    k = num 
+    D, I = index.search(decode_text.reshape(1, -1), k)
     
-    if num:
-
-        k = num  # Задайте количество выводимых результатов
-        top_similar_indices = np.argsort(cosine_similarities)[-k:][::-1]  # Получение индексов наиболее похожих предложений
-        top_similar_annotations = [df['annotation'].iloc[i] for i in top_similar_indices]  # Получение самих предложений
-
-        top_similar_images = [df['image_url'].iloc[i] for i in top_similar_indices]
-        images = [Image.open(BytesIO(requests.get(url).content)) for url in top_similar_images]  
-        top_similar_authors = [df['author'].iloc[i] for i in top_similar_indices] 
-        top_similar_title = [df['title'].iloc[i] for i in top_similar_indices]
-        top_cosine_similarities = [cosine_similarities[i] for i in top_similar_indices]
-    
-    # Отображение изображений и названий
-    for similarity, image, author, annotation, title in zip(top_cosine_similarities, images, top_similar_authors, top_similar_annotations,top_similar_title):
+    top_similar_indices = I[0]
+    top_similar_annotations = [df['annotation'].iloc[i] for i in top_similar_indices]
+    top_similar_images = [df['image_url'].iloc[i] for i in top_similar_indices]
+    images = [Image.open(BytesIO(requests.get(url).content)) for url in top_similar_images]
+    top_similar_authors = [df['author'].iloc[i] for i in top_similar_indices]
+    top_similar_title = [df['title'].iloc[i] for i in top_similar_indices]
+    top_similar_url = [df['page_url'].iloc[i] for i in top_similar_indices]
+    top_cosine_similarities = [1 - d / 2 for d in D[0]]  # Преобразование расстояний в косинусное сходство
+
+# Отображение изображений и названий
+    for similarity, image, author, annotation, title, url in zip(top_cosine_similarities, images, top_similar_authors, top_similar_annotations, top_similar_title, top_similar_url):
         col1, col2 = st.columns([3, 4]) 
         with col1:
             st.image(image, width=300)
@@ -67,6 +69,7 @@ if text and button:
             st.write(f"***Аннотация:*** {annotation}")
             similarity = float(similarity)
             st.write(f"***Cosine Similarity : {round(similarity, 3)}***")
+            st.write(f"***Ссылка на книгу : {url}***")
 
         st.markdown(
         "<hr style='border: 2px solid #000; margin-top: 10px; margin-bottom: 10px;'>",
@@ -76,3 +79,4 @@ if text and button:
 
 
 
+

requirements.txt

@@ -7,6 +7,7 @@ cachetools==5.3.2
 certifi==2023.7.22
 charset-normalizer==3.3.2
 click==8.1.7
+faiss-cpu==1.7.2
 filelock==3.13.1
 fsspec==2023.10.0
 gitdb==4.0.11