Find Related: Optimize: Avoid creating embeddings for text that has not changed

routers/tool_find_related.py

@@ -12,9 +12,9 @@ from sentence_transformers import SentenceTransformer, util
 from fastapi import APIRouter
 
 try:
-    from .utils_gitea import gitea_fetch_issues, gitea_json_issue_get
+    from .utils_gitea import gitea_fetch_issues, gitea_json_issue_get, gitea_issues_body_updated_at_get
 except:
-    from utils_gitea import gitea_fetch_issues, gitea_json_issue_get
+    from utils_gitea import gitea_fetch_issues, gitea_json_issue_get, gitea_issues_body_updated_at_get
 
 
 def _create_issue_string(title, body):
@@ -56,7 +56,8 @@ class EmbeddingContext:
     TOKEN_LEN_MAX_FOR_EMBEDDING = 512
     TOKEN_LEN_MAX_BALCKLIST = 2 * TOKEN_LEN_MAX_FOR_EMBEDDING
     ARRAY_CHUNK_SIZE = 4096
-    issue_attr_filter = {'number', 'title', 'body', 'state', 'updated_at'}
+    issue_attr_filter = {'number', 'title', 'body',
+                         'state', 'updated_at', 'created_at'}
     cache_path = "routers/tool_find_related_cache.pkl"
 
     # Set when creating the object
@@ -278,24 +279,36 @@ class EmbeddingContext:
             data['updated_at'] = date_new
 
 # autopep8: off
-            # WORKAROUND:
             # Consider that if the time hasn't changed, it's the same issue.
             issues = [issue for issue in issues if issue['updated_at'] != date_old]
 
             self.data_ensure_size(repo, int(issues[0]['number']))
 
-            texts_to_embed = self.create_strings_to_embbed(issues, black_list)
-            embeddings = self.encode(texts_to_embed)
+            updated_at = gitea_issues_body_updated_at_get(issues)
+            issues_to_embed = []
 
             for i, issue in enumerate(issues):
                 number = int(issue['number'])
-                data['titles'][number] = issue['title']
-                data['embeddings'][number] = embeddings[i]
                 if issue['state'] == 'open':
                     data['opened'][number] = True
                 if issue['state'] == 'closed':
                     data['closed'][number] = True
 
+                title_old = data['titles'][number]
+                if title_old != issue['title']:
+                    data['titles'][number] = issue['title']
+                    issues_to_embed.append(issue)
+                elif updated_at[i] >= date_old:
+                    issues_to_embed.append(issue)
+
+            if issues_to_embed:
+                texts_to_embed = self.create_strings_to_embbed(issues_to_embed, black_list)
+                embeddings = self.encode(texts_to_embed)
+
+                for i, issue in enumerate(issues_to_embed):
+                    number = int(issue['number'])
+                    data['embeddings'][number] = embeddings[i]
+
 # autopep8: on
         return data
 

routers/tool_find_related_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:178cabd52e35e69184b0e49a0bdae18478e99d1b5cec6f590840e7d7c65576d8
-size 723396066
+oid sha256:a181cc69d535d6502588e4c14bea367d74dfaca17a5602a23a72def479f592cc
+size 723433353

routers/utils_gitea.py

@@ -87,3 +87,24 @@ def gitea_fetch_issues(owner, repo, state='all', labels='', issue_attr_filter=No
     encoded_query_params = urllib.parse.urlencode(query_params)
     issues_url = f"{base_url}?{encoded_query_params}"
     return url_json_get_all_pages(issues_url, item_filter=issue_attr_filter, exclude=exclude, verbose=True)
+
+
+def gitea_issues_body_updated_at_get(issues, verbose=True):
+    def fetch_issue(issue):
+        number = issue['number']
+        if verbose:
+            print(f"Fetched issue #{number}")
+
+        json_data = url_json_get(
+            f"https://projects.blender.org/blender/blender/issues/{number}/content-history/list")
+        # Verify that the response contains the expected data before trying to access it
+        if json_data and json_data['results']:
+            return json_data['results'][0]['name'].split('datetime="')[1].split('"')[0]
+        else:
+            return issue['created_at']
+
+    with ThreadPoolExecutor() as executor:
+        futures = [executor.submit(fetch_issue, issue) for issue in issues]
+        all_results = [future.result() for future in as_completed(futures)]
+
+    return all_results