Add Utils for generate documentantion

Currently `utils/generate_blender_doc.py` is not used because
"mano-wii/BAAI_bge-base-en-v1.5-tunned-for-blender-issues" is not
trained to get good scores for terms.

routers/tool_find_related.py

@@ -11,14 +11,8 @@ from fastapi import APIRouter
 
 try:
     from .utils_gitea import gitea_fetch_issues, gitea_json_issue_get
-    from config import settings
 except:
-    import sys
     from utils_gitea import gitea_fetch_issues, gitea_json_issue_get
-    sys.path.append(os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..')))
-    from config import settings
-
 
 def _create_issue_string(title, body):
     cleaned_body = body.replace('\r', '')
@@ -76,6 +70,14 @@ class EmbeddingContext:
     def __init__(self):
         self.lock = threading.Lock()
 
+        try:
+            from config import settings
+        except:
+            import sys
+            sys.path.append(os.path.abspath(
+                os.path.join(os.path.dirname(__file__), '..')))
+            from config import settings
+
         config_type = settings.embedding_api
         model_name = settings.embedding_model
 
@@ -162,8 +164,9 @@ class EmbeddingContext:
             if tokens_len > self.TOKEN_LEN_MAX_BALCKLIST:
                 # Only use the first TOKEN_LEN_MAX tokens
                 black_list.add(int(issues[i]['number']))
-                texts_to_embed[i] = ' '.join(
-                    tokens[:self.TOKEN_LEN_MAX_BALCKLIST])
+                if self.config_type == 'openai':
+                    texts_to_embed[i] = ' '.join(
+                        tokens[:self.TOKEN_LEN_MAX_BALCKLIST])
 
         return texts_to_embed
 

routers/tool_find_related_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcfc6263dd6858337017723f80b66c7c8f657ad9675b0263ea30f3940291c3b6
-size 21219373
+oid sha256:1400db7078bac757625cc72f04d329a218638b3a267c22c2f20a2bba9a52e787
+size 21426824

utils/generate_blender_doc.py

@@ -0,0 +1,194 @@
+import os
+import sys
+import re
+from sentence_transformers import util
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+parent_dir = os.path.dirname(script_dir)
+sys.path.append(parent_dir)
+
+# autopep8: off
+from routers.tool_find_related import EMBEDDING_CTX
+# autopep8: on
+
+MANUAL_DIR = "D:/BlenderDev/blender-manual/manual/"
+BASE_URL = "https://docs.blender.org/manual/en/dev"
+
+
+def process_text(text):
+    # Remove repeated characters
+    text = re.sub(r'%{2,}', '', text)
+    text = re.sub(r'#{2,}', '', text)
+    text = re.sub(r'={3,}', '', text)
+    text = re.sub(r'\*{3,}', '', text)
+    text = re.sub(r'\^{3,}', '', text)
+    text = re.sub(r'-{3,}', '', text)
+
+    # Remove patterns ".. word:: " and ":word:"
+    text = re.sub(r'\.\. \S+', '', text)
+    text = re.sub(r':\w+:', '', text)
+
+    text = re.sub(r'(\s*\n\s*)+', '\n', text)
+    return text
+
+
+def parse_file(filedir, filename):
+    with open(os.path.join(filedir, filename), 'r', encoding='utf-8') as file:
+        content = file.read()
+
+    parsed_data = {}
+
+    if not filename.endswith('index.rst'):
+        body = content.strip()
+    else:
+        parts = content.split(".. toctree::")
+        body = parts[0].strip()
+
+        if len(parts) > 1:
+            parsed_data["toctree"] = {}
+            for part in parts[1:]:
+                toctree_entries = part.split('\n')
+                line = toctree_entries[0]
+                for entry in toctree_entries[1:]:
+                    entry = entry.strip()
+                    if not entry:
+                        continue
+
+                    if entry.startswith('/'):
+                        # relative path.
+                        continue
+
+                    if not entry.endswith('.rst'):
+                        continue
+
+                    if entry.endswith('/index.rst'):
+                        entry_name = entry[:-10]
+                        filedir_ = os.path.join(filedir, entry_name)
+                        filename_ = 'index.rst'
+                    else:
+                        entry_name = entry[:-4]
+                        filedir_ = filedir
+                        filename_ = entry
+
+                    parsed_data['toctree'][entry_name] = parse_file(
+                        filedir_, filename_)
+
+    processed_text = process_text(body)
+    tokens = EMBEDDING_CTX.model.tokenizer.tokenize(processed_text)
+    if len(tokens) > EMBEDDING_CTX.model.max_seq_length:
+        pass
+    # parsed_data['body'] = body
+    parsed_data['processed_text'] = processed_text
+    parsed_data['n_tokens'] = len(tokens)
+
+    return parsed_data
+
+
+# Function to split the text into chunks of a maximum number of tokens
+def split_into_many(text, max_tokens):
+
+    # Split the text into sentences
+    paragraphs = text.split('.\n')
+
+    # Get the number of tokens for each sentence
+    n_tokens = [len(EMBEDDING_CTX.model.tokenizer.tokenize(" " + sentence))
+                for sentence in paragraphs]
+
+    chunks = []
+    tokens_so_far = 0
+    chunk = []
+
+    # Loop through the sentences and tokens joined together in a tuple
+    for sentence, token in zip(paragraphs, n_tokens):
+
+        # If the number of tokens so far plus the number of tokens in the current sentence is greater
+        # than the max number of tokens, then add the chunk to the list of chunks and reset
+        # the chunk and tokens so far
+        if tokens_so_far + token > max_tokens:
+            chunks.append((".\n".join(chunk) + ".", tokens_so_far))
+            chunk = []
+            tokens_so_far = 0
+
+        # If the number of tokens in the current sentence is greater than the max number of
+        # tokens, go to the next sentence
+        if token > max_tokens:
+            continue
+
+        # Otherwise, add the sentence to the chunk and add the number of tokens to the total
+        chunk.append(sentence)
+        tokens_so_far += token + 1
+
+    if chunk:
+        chunks.append((".\n".join(chunk) + ".", tokens_so_far))
+
+    return chunks
+
+
+def get_texts(data, path):
+    result = []
+    processed_texts = [data['processed_text']]
+    processed_tokens = [data['n_tokens']]
+    max_tokens = EMBEDDING_CTX.model.max_seq_length
+
+    data_ = data
+    for key in path:
+        data_ = data_['toctree'][key]
+        processed_texts.append(data_['processed_text'])
+        processed_tokens.append(data_['n_tokens'])
+
+    if processed_tokens[-1] > max_tokens:
+        chunks = split_into_many(processed_texts[-1], max_tokens)
+    else:
+        chunks = [(processed_texts[-1], processed_tokens[-1])]
+
+    for text, n_tokens in chunks:
+        # Add context to the text if we have space
+        for i in range(len(processed_texts) - 2, -1, -1):
+            n_tokens_parent = processed_tokens[i]
+            if n_tokens + n_tokens_parent >= max_tokens:
+                break
+
+            text_parent = processed_texts[i]
+            text = text_parent + '\n' + text
+            n_tokens += n_tokens_parent
+
+        result.append([path, text])
+
+    try:
+        for key in data_['toctree'].keys():
+            result.extend(get_texts(data, path + [key]))
+    except KeyError:
+        pass
+
+    return result
+
+
+def _sort_similarity(chunks, embeddings, text_to_search, limit):
+    results = []
+
+    query_emb = EMBEDDING_CTX.encode([text_to_search])
+    ret = util.semantic_search(
+        query_emb, embeddings, top_k=limit, score_function=util.dot_score)
+
+    for score in ret[0]:
+        corpus_id = score['corpus_id']
+        chunk = chunks[corpus_id]
+        path = chunk[0]
+        results.append(path)
+
+    return results
+
+
+if __name__ == '__main__':
+    # path = 'addons/3d_view'
+    data = parse_file(MANUAL_DIR, 'index.rst')
+    data['toctree']["copyright"] = parse_file(MANUAL_DIR, 'copyright.rst')
+
+    # Create a list to store the text files
+    chunks = []
+    chunks.extend(get_texts(data, []))
+
+    embeddings = EMBEDDING_CTX.encode([text for path, text in chunks])
+
+    result = _sort_similarity(chunks, embeddings, "Set Snap Base", 50)
+    print(result)

utils/generate_bpy_doc.py

@@ -0,0 +1,224 @@
+import bpy
+import inspect
+import time
+import pickle
+import mathutils
+import os
+import bpy_types
+import addon_utils
+import sys
+
+INFO_MEMBER = "__info"
+
+
+def get_info(name="", descr="", bases=None):
+    return {"name": name,
+            "descr":  descr,
+            "bases": bases}
+
+##################################################################
+
+
+g_bpy_types = {}
+
+
+def doc_from_bpy_struct(bl_rna):
+    bases = []
+    try:
+        base = bl_rna.base
+        while base:
+            bases.append(type(base).__name__)
+            base = base.base
+    except:
+        if not bases:
+            bases = None
+
+    return get_info(name=bl_rna.name, descr=bl_rna.description, bases=bases)
+
+
+def bpy_type_first_step(bpy_type):
+    def is_member_from_base_class(bpy_type, identifier):
+        if identifier in bpy.types.ID.bl_rna.properties:
+            return True
+
+        bases = bpy_type.mro()[1:]
+        for base in bases:
+            if not hasattr(base, "bl_rna"):
+                continue
+            if identifier in base.bl_rna.properties:
+                return True
+        return False
+
+    info = doc_from_bpy_struct(bpy_type.bl_rna)
+    data = {INFO_MEMBER: info}
+    for prop in bpy_type.bl_rna.properties:
+        identifier = prop.identifier
+        if is_member_from_base_class(bpy_type, identifier):
+            continue
+        if prop.type == 'POINTER':
+            srna_type = prop.fixed_type.identifier
+            try:
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = pointer_type
+            except Exception:
+                pass
+            continue
+        if prop.type == 'COLLECTION':
+            if prop.srna:
+                srna_type = prop.srna.identifier
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = pointer_type
+            elif srna_type := prop.fixed_type.identifier:
+                pointer_type = getattr(bpy.types, srna_type)
+                data[identifier] = [pointer_type]
+            continue
+
+        info_member = doc_from_bpy_struct(prop)
+        data[identifier] = {INFO_MEMBER: info_member}
+
+    return data
+
+
+def bpy_types_first_step():
+    global g_bpy_types
+    for bpy_type_name in dir(bpy.types):
+        bpy_type = getattr(bpy.types, bpy_type_name)
+        if not hasattr(bpy_type, "bl_rna"):
+            continue
+        g_bpy_types[bpy_type] = bpy_type_first_step(bpy_type)
+
+
+def bpy_types_second_step():
+    global g_bpy_types
+    for bpy_type, map in g_bpy_types.items():
+        for key, val in map.items():
+            if hasattr(val, "bl_rna"):
+                map[key] = g_bpy_types[val]
+            elif isinstance(val, list):
+                val[0] = g_bpy_types[val[0]]
+
+
+##################################################################
+
+bases_builtin = {int, bool, float, str, bytes, tuple, list,
+                 set, dict, mathutils.Vector, mathutils.Color, type(None)}
+
+
+def is_member_inherited(obj, member):
+    mro_bases = inspect.getmro(type(obj))
+    mro_bases_set = set(mro_bases)
+    intersection = mro_bases_set.intersection(bases_builtin)
+    for base in intersection:
+        if hasattr(base, member):
+            return True
+    return False
+
+
+def get_doc_recursive(parent, member):
+    ob = getattr(parent, member)
+    member_info = getattr(type(parent), member, ob)
+    if type(member_info) in bases_builtin or member == "bpy_func":
+        descr = type(member_info).__name__
+        return {INFO_MEMBER: get_info(descr=descr)}
+
+    if hasattr(type(ob), "bl_rna"):
+        return g_bpy_types[type(ob)]
+
+    if "bl_rna" in dir(ob):
+        return g_bpy_types[ob]
+
+    result = {}
+    descr = member_info.__doc__ if member_info.__doc__ else type(ob).__name__
+    result[INFO_MEMBER] = get_info(descr=descr)
+
+    for name in dir(ob):
+        if name.startswith("_"):
+            continue
+        if is_member_inherited(ob, name):
+            continue
+
+        ob_member = getattr(ob, name, None)
+        if ob_member == parent:
+            descr = type(parent).__name__
+            result[name] = {INFO_MEMBER: get_info(descr=descr)}
+            continue
+        if ob_member == os:
+            continue
+        if ob_member == bpy:
+            continue
+        if ob_member == bpy_types:
+            continue
+        if ob_member == addon_utils:
+            continue
+        if ob_member == sys:
+            continue
+        if name == "addon_install":
+            # This raises a Error
+            continue
+
+        result[name] = get_doc_recursive(ob, name)
+    return result
+
+
+##################################################################
+
+def print_doc_recursive(map, indent, name, max_step=3):
+    time.sleep(.5)
+    prefix = indent * '|'
+    print(prefix + name)
+    for key, val in map.items():
+        if key == INFO_MEMBER:
+            print(prefix + val.replace('\n', '\n' + prefix) + '\n' + prefix)
+        elif indent < max_step:
+            name_next = name + '.' + key
+            if isinstance(val, list):
+                print_doc_recursive(val[0], indent + 1,
+                                    name_next + "[0]", max_step=max_step)
+            else:
+                print_doc_recursive(
+                    val, indent + 1, name_next, max_step=max_step)
+
+
+def main():
+    print("-------------------------------------------------------------")
+    bpy_types_first_step()
+    bpy_types_second_step()
+
+    members = (
+        "app",
+        "context",
+        "data",
+        "msgbus",
+        "ops",
+        "path",
+        "props",
+        "types",
+        "utils",
+    )
+
+    result = {
+        "bpy": {INFO_MEMBER: get_info(descr=bpy.__doc__)},
+        "__info": {"bases": None},
+    }
+    for member in members:
+        result["bpy"][member] = get_doc_recursive(bpy, member)
+
+    # Reference some types at the beginning
+    result["bpy_struct"] = result["bpy"]["types"]["bpy_struct"]
+    result["bpy_types"] = result["bpy"]["types"]
+
+    if False:
+        print(result["bpy"]["props"]["BoolProperty"])
+        return
+
+    # print_doc_recursive(result, 1, "bpy")
+    bpy_doc_dir = "D:/Dev/function-calling/routersbpy_doc_v41.pkl"
+    with open(bpy_doc_dir, "wb") as file:
+        # print(result["types"]["bpy_func"])
+        pickle.dump(result, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+    print(f"File '{bpy_doc_dir}' has been updated.")
+
+
+if __name__ == '__main__':
+    main()