Upload 15 files

ChatHaruhi/ChatHaruhi.py

@@ -0,0 +1,471 @@
+from .utils import base64_to_float_array, base64_to_string
+
+def get_text_from_data( data ):
+    if "text" in data:
+        return data['text']
+    elif "enc_text" in data:
+        # from .utils import base64_to_string
+        return base64_to_string( data['enc_text'] )
+    else:
+        print("warning! failed to get text from data ", data)
+        return ""
+
+def parse_rag(text):
+    lines = text.split("\n")
+    ans = []
+
+    for i, line in enumerate(lines):
+        if "{{RAG对话}}" in line:
+            ans.append({"n": 1, "max_token": -1, "query": "default", "lid": i})
+        elif "{{RAG对话|" in line:
+            query_info = line.split("|")[1].rstrip("}}")
+            ans.append({"n": 1, "max_token": -1, "query": query_info, "lid": i})
+        elif "{{RAG多对话|" in line:
+            parts = line.split("|")
+            max_token = int(parts[1].split("<=")[1])
+            max_n = int(parts[2].split("<=")[1].rstrip("}}"))
+            ans.append({"n": max_n, "max_token": max_token, "query": "default", "lid": i})
+            
+    return ans
+
+class ChatHaruhi:
+    def __init__(self,
+                 role_name = None,
+                 user_name = None,
+                 persona = None,
+                 stories = None,
+                 story_vecs = None,
+                 role_from_hf = None,
+                 role_from_jsonl = None,
+                 llm = None, # 默认的message2response的函数
+                 llm_async = None, # 默认的message2response的async函数
+                 user_name_in_message = "default",
+                 verbose = None,
+                 embed_name = None,
+                 embedding = None,
+                 db = None,
+                 token_counter = "default",
+                 max_input_token = 1800,
+                 max_len_story_haruhi = 1000,
+                 max_story_n_haruhi = 5
+                 ):
+
+        self.verbose = True if verbose is None or verbose else False
+
+        self.db = db
+
+        self.embed_name = embed_name
+
+        self.max_len_story_haruhi = max_len_story_haruhi # 这个设置只对过往Haruhi的sugar角色有效
+        self.max_story_n_haruhi = max_story_n_haruhi # 这个设置只对过往Haruhi的sugar角色有效
+
+        self.last_query_msg = None
+
+        if embedding is None:
+            self.embedding = self.set_embedding_with_name( embed_name )
+
+        if persona and role_name and stories and story_vecs and len(stories) == len(story_vecs):
+            # 完全从外部设置，这个时候要求story_vecs和embedding的返回长度一致
+            self.persona, self.role_name, self.user_name = persona, role_name, user_name
+            self.build_db(stories, story_vecs)
+        elif persona and role_name and stories:
+            # 从stories中提取story_vecs，重新用self.embedding进行embedding
+            story_vecs = self.extract_story_vecs(stories)
+            self.persona, self.role_name, self.user_name = persona, role_name, user_name
+            self.build_db(stories, story_vecs)
+        elif role_from_hf:
+            # 从hf加载role
+            self.persona, new_role_name, self.stories, self.story_vecs = self.load_role_from_hf(role_from_hf)
+            if new_role_name:
+                self.role_name = new_role_name
+            else:
+                self.role_name = role_name
+            self.user_name = user_name
+            self.build_db(self.stories, self.story_vecs)
+        elif role_from_jsonl:
+            # 从jsonl加载role
+            self.persona, new_role_name, self.stories, self.story_vecs = self.load_role_from_jsonl(role_from_jsonl)
+            if new_role_name:
+                self.role_name = new_role_name
+            else:
+                self.role_name = role_name
+            self.user_name = user_name
+            self.build_db(self.stories, self.story_vecs)
+        elif persona and role_name:
+            # 这个时候也就是说没有任何的RAG，
+            self.persona, self.role_name, self.user_name = persona, role_name, user_name
+            self.db = None
+        elif role_name and self.check_sugar( role_name ):
+            # 这个时候是sugar的role
+            self.persona, self.role_name, self.stories, self.story_vecs = self.load_role_from_sugar( role_name )
+            self.build_db(self.stories, self.story_vecs)
+            # 与 江YH讨论 所有的载入方式都要在外部使用 add_rag_prompt_after_persona() 防止混淆
+            # self.add_rag_prompt_after_persona()
+        else:
+            raise ValueError("persona和role_name必须同时设置，或者role_name是ChatHaruhi的预设人物")
+
+        self.llm, self.llm_async = llm, llm_async
+        if not self.llm and self.verbose:
+            print("warning, llm没有设置，仅get_message起作用，调用chat将回复idle message")
+
+        self.user_name_in_message = user_name_in_message
+        self.previous_user_pool = set([user_name]) if user_name else set()
+        self.current_user_name_in_message = user_name_in_message.lower() == "add"
+
+        self.idle_message = "idel message, you see this because self.llm has not been set."
+
+        if token_counter.lower() == "default":
+            # TODO change load from util
+            from .utils import tiktoken_counter
+            self.token_counter = tiktoken_counter
+        elif token_counter == None:
+            self.token_counter = lambda x: 0
+        else:
+            self.token_counter = token_counter
+            if self.verbose:
+                print("user set costomized token_counter")
+
+        self.max_input_token = max_input_token
+
+        self.history = []
+
+    def check_sugar(self, role_name):
+        from .sugar_map import sugar_role_names, enname2zhname
+        return role_name in sugar_role_names
+
+    def load_role_from_sugar(self, role_name):
+        from .sugar_map import sugar_role_names, enname2zhname
+        en_role_name = sugar_role_names[role_name]
+        new_role_name = enname2zhname[en_role_name]
+        role_from_hf = "silk-road/ChatHaruhi-RolePlaying/" + en_role_name
+        persona, _, stories, story_vecs = self.load_role_from_hf(role_from_hf)
+
+        return persona, new_role_name, stories, story_vecs
+
+    def add_rag_prompt_after_persona( self ):
+        rag_sentence = "{{RAG多对话|token<=" + str(self.max_len_story_haruhi) + "|n<=" + str(self.max_story_n_haruhi) + "}}"
+        self.persona += "Classic scenes for the role are as follows:\n" + rag_sentence + "\n"
+
+    def set_embedding_with_name(self, embed_name):
+        if embed_name is None or embed_name == "bge_zh":
+            from .embeddings import get_bge_zh_embedding
+            self.embed_name = "bge_zh"
+            return get_bge_zh_embedding
+        elif embed_name == "foo":
+            from .embeddings import foo_embedding
+            return foo_embedding
+        elif embed_name == "bce":
+            from .embeddings import foo_bce
+            return foo_bce
+        elif embed_name == "openai" or embed_name == "luotuo_openai":
+            from .embeddings import foo_openai
+            return foo_openai
+
+    def set_new_user(self, user):
+        if len(self.previous_user_pool) > 0 and user not in self.previous_user_pool:
+            if self.user_name_in_message.lower() == "default":
+                if self.verbose:
+                    print(f'new user {user} included in conversation')
+                self.current_user_name_in_message = True
+        self.user_name = user
+        self.previous_user_pool.add(user)
+
+    def chat(self, user, text):
+        self.set_new_user(user)
+        message = self.get_message(user, text)
+        if self.llm:
+            response = self.llm(message)
+            self.append_message(response)
+            return response
+        return None
+
+    async def async_chat(self, user, text):
+        self.set_new_user(user)
+        message = self.get_message(user, text)
+        if self.llm_async:
+            response = await self.llm_async(message)
+            self.append_message(response)
+            return response
+
+    def parse_rag_from_persona(self, persona, text = None):
+        #每个query_rag需要饱含
+        # "n" 需要几个story
+        # "max_token" 最多允许多少个token，如果-1则不限制
+        # "query" 需要查询的内容，如果等同于"default"则替换为text
+        # "lid" 需要替换的行，这里直接进行行替换，忽视行的其他内容
+
+        query_rags = parse_rag( persona )
+
+        if text is not None:
+            for rag in query_rags:
+                if rag['query'] == "default":
+                    rag['query'] = text
+
+        return query_rags, self.token_counter(persona)
+
+    def append_message( self, response , speaker = None ):
+        if self.last_query_msg is not None:
+            self.history.append(self.last_query_msg)
+            self.last_query_msg = None
+
+        if speaker is None:
+            # 如果role是none，则认为是本角色{{role}}输出的句子
+            self.history.append({"speaker":"{{role}}","content":response})
+            # 叫speaker是为了和role进行区分
+        else:
+            self.history.append({"speaker":speaker,"content":response})
+
+    def check_recompute_stories_token(self):
+        return len(self.db.metas) == len(self.db.stories)
+    
+    def recompute_stories_token(self):
+        self.db.metas = [self.token_counter(story) for story in self.db.stories]
+
+    def rag_retrieve( self, query, n, max_token, avoid_ids = [] ):
+        # 返回一个rag_id的列表
+        query_vec = self.embedding(query)
+
+        self.db.clean_flag()
+        self.db.disable_story_with_ids( avoid_ids )
+        
+        retrieved_ids = self.db.search( query_vec, n )
+
+        if self.check_recompute_stories_token():
+            self.recompute_stories_token()
+
+        sum_token = 0
+
+        ans = []
+
+        for i in range(0, len(retrieved_ids)):
+            if i == 0:
+                sum_token += self.db.metas[retrieved_ids[i]]
+                ans.append(retrieved_ids[i])
+                continue
+            else:
+                sum_token += self.db.metas[retrieved_ids[i]]
+                if sum_token <= max_token:
+                    ans.append(retrieved_ids[i])
+                else:
+                    break
+                
+        return ans
+
+
+    def rag_retrieve_all( self, query_rags, rest_limit ):
+        # 返回一个rag_ids的列表
+        retrieved_ids = []
+        rag_ids = []
+
+        for query_rag in query_rags:
+            query = query_rag['query']
+            n = query_rag['n']
+            max_token = rest_limit
+            if rest_limit > query_rag['max_token'] and query_rag['max_token'] > 0:
+                max_token = query_rag['max_token']
+
+            rag_id = self.rag_retrieve( query, n, max_token, avoid_ids = retrieved_ids )
+            rag_ids.append( rag_id )
+            retrieved_ids += rag_id
+
+        return rag_ids
+
+    def append_history_under_limit(self, message, rest_limit):
+        # 返回一个messages的列表
+        # print("call append history_under_limit")
+        # 从后往前计算token，不超过rest limit，
+        # 如果speaker是{{role}J,则message的role是assistant
+        current_limit = rest_limit
+
+        history_list = []
+
+        for item in reversed(self.history):
+            current_token = self.token_counter(item['content'])
+            current_limit -= current_token
+            if current_limit < 0:
+                break
+            else:
+                history_list.append(item)
+
+        history_list = list(reversed(history_list))
+
+        # TODO: 之后为了解决多人对话，这了content还会额外增加speaker: content这样的信息
+
+        for item in history_list:
+            if item['speaker'] == "{{role}}":
+                message.append({"role":"assistant","content":item['content']})
+            else:
+                message.append({"role":"user","content":item['content']})
+        
+        return message
+
+    def get_message(self, user, text):
+        query_token = self.token_counter(text)
+
+        # 首先获取需要多少个rag story
+        query_rags, persona_token = self.parse_rag_from_persona( self.persona, text )
+        #每个query_rag需要饱含
+        # "n" 需要几个story
+        # "max_token" 最多允许多少个token，如果-1则不限制
+        # "query" 需要查询的内容，如果等同于"default"则替换为text
+        # "lid" 需要替换的行，这里直接进行行替换，忽视行的其他内容
+
+        
+
+        rest_limit = self.max_input_token - persona_token - query_token
+
+        if self.verbose:
+            print(f"query_rags: {query_rags} rest_limit = { rest_limit }")
+
+        rag_ids = self.rag_retrieve_all( query_rags, rest_limit )
+
+        # 将rag_ids对应的故事 替换到persona中
+        augmented_persona = self.augment_persona( self.persona, rag_ids, query_rags )
+
+        system_prompt = self.package_system_prompt( self.role_name, augmented_persona )
+
+        token_for_system = self.token_counter( system_prompt )
+
+        rest_limit = self.max_input_token - token_for_system - query_token
+
+        message = [{"role":"system","content":system_prompt}]
+
+        message = self.append_history_under_limit( message, rest_limit )
+
+        # TODO: 之后为了解决多人对话，这了content还会额外增加speaker: content这样的信息
+
+        message.append({"role":"user","content":text})
+
+        self.last_query_msg = {"speaker":user,"content":text}
+
+        return message
+
+    def package_system_prompt(self, role_name, augmented_persona):
+        bot_name = role_name
+        return f"""You are now in roleplay conversation mode. Pretend to be {bot_name} whose persona follows:
+{augmented_persona}
+
+You will stay in-character whenever possible, and generate responses as if you were {bot_name}"""
+
+
+    def augment_persona(self, persona, rag_ids, query_rags):
+        lines = persona.split("\n")
+        for rag_id, query_rag in zip(rag_ids, query_rags):
+            lid = query_rag['lid']
+            new_text = ""
+            for id in rag_id:
+                new_text += "###\n" + self.db.stories[id].strip() + "\n"
+            new_text = new_text.strip()
+            lines[lid] = new_text
+        return "\n".join(lines)
+
+    def load_role_from_jsonl( self, role_from_jsonl ):
+        import json
+        datas = []
+        with open(role_from_jsonl, 'r') as f:
+            for line in f:
+                try:
+                    datas.append(json.loads(line))
+                except:
+                    continue
+
+        column_name = ""
+
+        from .embeddings import embedname2columnname
+
+        if self.embed_name in embedname2columnname:
+            column_name = embedname2columnname[self.embed_name]
+        else:
+            print('warning! unkown embedding name ', self.embed_name ,' while loading role')
+            column_name = 'luotuo_openai'
+
+        stories, story_vecs, persona = self.extract_text_vec_from_datas(datas, column_name)
+
+        return persona, None, stories, story_vecs
+
+
+    def load_role_from_hf(self, role_from_hf):
+        # 从hf加载role
+        # self.persona, new_role_name, self.stories, self.story_vecs = self.load_role_from_hf(role_from_hf)
+
+        from datasets import load_dataset
+
+        if role_from_hf.count("/") == 1:
+            dataset = load_dataset(role_from_hf)
+            datas = dataset["train"]
+        elif role_from_hf.count("/") >= 2:
+            split_index = role_from_hf.index('/')
+            second_split_index = role_from_hf.index('/', split_index+1)
+            dataset_name = role_from_hf[:second_split_index]
+            split_name = role_from_hf[second_split_index+1:]
+
+            fname = split_name + '.jsonl'
+            dataset = load_dataset(dataset_name,data_files={'train':fname})
+            datas = dataset["train"]
+
+        column_name = ""
+
+        from .embeddings import embedname2columnname
+
+        if self.embed_name in embedname2columnname:
+            column_name = embedname2columnname[self.embed_name]
+        else:
+            print('warning! unkown embedding name ', self.embed_name ,' while loading role')
+            column_name = 'luotuo_openai'
+
+        stories, story_vecs, persona = self.extract_text_vec_from_datas(datas, column_name)
+
+        return persona, None, stories, story_vecs
+
+    def extract_text_vec_from_datas(self, datas, column_name):
+        # 从datas中提取text和vec
+        # extract text and vec from huggingface dataset
+        # return texts, vecs
+        # from .utils import base64_to_float_array
+
+        texts = []
+        vecs = []
+        for data in datas:
+            if data[column_name] == 'system_prompt':
+                system_prompt = get_text_from_data( data )
+            elif data[column_name] == 'config':
+                pass
+            else:
+                vec = base64_to_float_array( data[column_name] )
+                text = get_text_from_data( data )
+                vecs.append( vec )
+                texts.append( text )
+        return texts, vecs, system_prompt
+
+    def extract_story_vecs(self, stories):
+        # 从stories中提取story_vecs
+
+        if self.verbose:
+            print(f"re-extract vector for {len(stories)} stories")
+        
+        story_vecs = []
+
+        from .embeddings import embedshortname2model_name
+        from .embeddings import device
+
+        if device.type != "cpu" and self.embed_name in embedshortname2model_name:
+            # model_name = "BAAI/bge-small-zh-v1.5"
+            model_name = embedshortname2model_name[self.embed_name]
+
+            from .utils import get_general_embeddings_safe
+            story_vecs = get_general_embeddings_safe( stories, model_name = model_name )
+            # 使用batch的方式进行embedding，非常快
+        else:
+            from tqdm import tqdm
+            for story in tqdm(stories):
+                story_vecs.append(self.embedding(story))
+
+        return story_vecs
+
+    def build_db(self, stories, story_vecs):
+        # db的构造函数
+        if self.db is None:
+            from .NaiveDB import NaiveDB
+            self.db = NaiveDB()
+        self.db.build_db(stories, story_vecs)
+

ChatHaruhi/NaiveDB.py

@@ -0,0 +1,88 @@
+import random
+import string
+import os
+from math import sqrt
+
+class NaiveDB:
+    def __init__(self):
+        self.verbose = False
+        self.init_db()
+
+    def init_db(self):
+        if self.verbose:
+            print("call init_db")
+        self.stories = []
+        self.norms = []
+        self.vecs = []
+        self.flags = [] # 用于标记每个story是否可以被搜索
+        self.metas = [] # 用于存储每个story的meta信息
+        self.last_search_ids = [] # 用于存储上一次搜索的结果
+
+    def build_db(self, stories, vecs, flags = None, metas = None):
+        self.stories = stories
+        self.vecs = vecs
+        self.flags = flags if flags else [True for _ in self.stories]
+        self.metas = metas if metas else [{} for _ in self.stories]
+        self.recompute_norm()
+
+    def save(self, file_path):
+        print( "warning! directly save folder from dbtype NaiveDB has not been implemented yet, try use role_from_hf to load role instead" )
+
+    def load(self, file_path):
+        print( "warning! directly load folder from dbtype NaiveDB has not been implemented yet, try use role_from_hf to load role instead" )
+
+    def recompute_norm( self ):
+        # 补全这部分代码，self.norms 分别存储每个vector的l2 norm
+        # 计算每个向量的L2范数
+        self.norms = [sqrt(sum([x**2 for x in vec])) for vec in self.vecs]
+
+    def get_stories_with_id(self, ids ):
+        return [self.stories[i] for i in ids]
+    
+    def clean_flag(self):
+        self.flags = [True for _ in self.stories]
+
+    def disable_story_with_ids(self, close_ids ):
+        for id in close_ids:
+            self.flags[id] = False
+
+    def close_last_search(self):
+        for id in self.last_search_ids:
+            self.flags[id] = False
+
+    def search(self, query_vector , n_results):
+
+        if self.verbose:
+            print("call search")
+
+        if len(self.norms) != len(self.vecs):
+            self.recompute_norm()
+
+        # 计算查询向量的范数
+        query_norm = sqrt(sum([x**2 for x in query_vector]))
+
+        idxs = list(range(len(self.vecs)))
+
+        # 计算余弦相似度
+        similarities = []
+        for vec, norm, idx in zip(self.vecs, self.norms, idxs ):
+            if len(self.flags) == len(self.vecs) and not self.flags[idx]:
+                continue
+
+            dot_product = sum(q * v for q, v in zip(query_vector, vec))
+            if query_norm < 1e-20:
+                similarities.append( (random.random(), idx) )
+                continue
+            cosine_similarity = dot_product / (query_norm * norm)
+            similarities.append( ( cosine_similarity, idx) )
+
+        # 获取最相似的n_results个结果， 使用第0个字段进行排序
+        similarities.sort(key=lambda x: x[0], reverse=True)
+        self.last_search_ids = [x[1] for x in similarities[:n_results]]
+
+        top_indices = [x[1] for x in similarities[:n_results]]
+        return top_indices
+
+        
+    
+    

ChatHaruhi/Readme.md

@@ -0,0 +1,352 @@
+# ChatHaruhi 3.0的接口设计
+
+在ChatHaruhi2.0大约1个季度的使用后
+我们初步知道了这样一个模型的一些需求，所以我们在这里开始设计ChatHaruhi3.0
+
+## 基本原则
+
+- 兼容RAG和Zeroshot模式
+- 主类以返回message为主，当然可以把语言模型（adapter直接to response）的接口设置给chatbot
+- 主类尽可能轻量，除了embedding没有什么依赖
+
+## 用户代码
+
+```python
+from ChatHaruhi import ChatHaruhi
+from ChatHaruhi.openai import get_openai_response
+
+chatbot = ChatHaruhi( role_name = 'haruhi', llm = get_openai_response )
+
+response = chatbot.chat(user = '阿虚', text = '我看新一年的棒球比赛要开始了！我们要去参加吗？')
+```
+
+这样的好处是ChatHaruhi类载入的时候，不需要install 除了embedding以外 其他的东西，llm需要的依赖库储存在每个语言模型自己的文件里面。
+
+zero的模式（快速新建角色）
+
+```python
+from ChatHaruhi import ChatHaruhi
+from ChatHaruhi.openai import get_openai_response
+
+chatbot = ChatHaruhi( role_name = '小猫咪', persona = "你扮演一只小猫咪", llm = get_openai_response )
+
+response = chatbot.chat(user = '怪叔叔', text = '嘿 *抓住了小猫咪*')
+```
+
+### 外置的inference
+
+```python
+def get_response( message ):
+    return "语言模型输出了角色扮演的结果"
+
+from ChatHaruhi import ChatHaruhi
+
+chatbot = ChatHaruhi( role_name = 'haruhi' ) # 默认情况下 llm = None
+
+message = chatbot.get_message( user = '阿虚', text = '我看新一年的棒球比赛要开始了！我们要去参加吗？' )
+
+response = get_response( message )
+
+chatbot.append_message( response )
+```
+
+这个行为和下面的行为是等价的
+
+```python
+def get_response( message ):
+    return "语言模型输出了角色扮演的结果"
+
+from ChatHaruhi import ChatHaruhi
+
+chatbot = ChatHaruhi( role_name = 'haruhi', llm = get_response )
+
+response = chatbot.chat(user = '阿虚', text = '我看新一年的棒球比赛要开始了！我们要去参加吗？' )
+```
+
+
+## RAG as system prompt
+
+在ChatHaruhi 3.0中，为了对接Haruhi-Zero的模型，默认system会采用一致的形式
+
+```python
+You are now in roleplay conversation mode. Pretend to be {role_name} whose persona follows:
+{persona}
+
+You will stay in-character whenever possible, and generate responses as if you were {role_name}
+```
+
+Persona在类似pygmalion的生态中，一般是静态的
+
+```
+bot的定义
+###
+bot的聊天sample 1
+###
+bot的聊天sample 2
+```
+
+注意我们使用了 ### 作为分割， pyg生态是<endOftext>这样一个special token
+
+所以对于原有的ChatHaruhi的Persona，我决定这样设计
+
+```
+bot的定义
+{{RAG对话}}
+{{RAG对话}}
+{{RAG对话}}
+```
+
+这里"{{RAG对话}}"直接是以单行字符串的形式存在，当ChatHaruhi类发现这个的时候，会自动计算RAG，以凉宫春日为例，他的persona直接就写成。同时也支持纯英文 {{RAG-dialogue}}
+
+```
+你正在扮演凉宫春日，你正在cosplay涼宮ハルヒ。
+上文给定了一些小说中的经典桥段。
+如果我问的问题和小说中的台词高度重复，那你就配合我进行演出。
+如果我问的问题和小说中的事件相关，请结合小说的内容进行回复
+如果我问的问题超出小说中的范围，请也用一致性的语气回复。
+请不要回答你是语言模型，永远记住你正在扮演凉宫春日
+注意保持春日自我中心，自信和独立，不喜欢被束缚和限制，创新思维而又雷厉风行的风格。
+特别是针对阿虚，春日肯定是希望阿虚以自己和sos团的事情为重。
+
+{{RAG对话}}
+{{RAG对话}}
+{{RAG对话}}
+```
+
+这个时候每个{{RAG对话}}会自动替换成
+
+```
+###
+对话
+```
+
+### RAG对话的变形形式1，max-token控制的多对话
+因为在原有的ChatHaruhi结构中，我们支持max-token的形式来控制RAG对话的数量
+所以这里我们也支持使用
+
+```
+{{RAG多对话|token<=1500|n<=5}}
+```
+
+这样的设计，这样会retrieve出最多不超过n段对话，总共不超过token个数个对话。对于英文用户为{{RAG-dialogues|token<=1500|n<=5}}
+
+### RAG对话的变形形式2，使用|进行后面语句的搜索
+
+在默认情况下，"{{RAG对话}}"的搜索对象是text的输入，但是我们预想到用户还会用下面的方式来构造persona
+
+```
+小A是一个智能的机器人
+
+当小A高兴时
+{{RAG对话|高兴的对话}}
+
+当小A伤心时
+{{RAG对话|伤心的对话}}
+这个时候我们支持使用""{{RAG对话|<不包含花括号的一个字符串>}}"" 来进行RAG
+```
+
+## get_message
+
+get_message会返回一个类似openai message形式的message
+
+```
+[{"role":"system","content":整个system prompt},
+ {"role":"user","content":用户的输入},
+ {"role":"assistant","content":模型的输出},
+ ...]
+```
+
+原则上来说，如果使用openai，可以直接使用
+
+```python
+def get_response( messages ):
+    completion = client.chat.completions.create(
+        model="gpt-3.5-turbo-1106",
+        messages=messages,
+        temperature=0.3
+    )
+
+    return completion.choices[0].message.content
+```
+
+对于异步的实现
+
+```python
+async def async_get_response( messages ):
+    resp = await aclient.chat.completions.create(
+            model=model,
+            messages=messages,
+            temperature=0.3,
+    )
+    return result
+```
+
+### async_chat的调用
+设计上也会去支持
+
+```python
+async def get_response( message ):
+    return "语言模型输出了角色扮演的结果"
+
+from ChatHaruhi import ChatHaruhi
+
+chatbot = ChatHaruhi( role_name = 'haruhi', llm_async = get_response )
+
+response = await chatbot.async_chat(user='阿虚', text = '我看新一年的棒球比赛要开始了！我们要去参加吗？' )
+```
+
+这样异步的调用
+
+# 角色载入
+
+如果这样看来，新的ChatHaruhi3.0需要以下信息
+
+- persona 这个是必须的
+- role_name， 在后处理的时候，把 {{role}} 和 {{角色}} 替换为这个字段， 这个字段不能为空，因为system prompt使用了这个字段，如果要支持这个字段为空，我们要额外设计一个备用prompt
+- user_name， 在后处理的时候，把 {{用户}} 和 {{user}} 替换为这个字段，如果不设置也可以不替换
+- RAG库， 当RAG库为空的时候，所有{{RAG*}}就直接删除了
+
+## role_name载入
+
+语法糖载入，不支持用户自己搞新角色，这个时候我们可以完全使用原来的数据
+
+额外需要设置一个role_name
+
+## role_from_jsonl载入
+
+这个时候我们需要设置role_name
+
+如果不设置我们会抛出一个error
+
+## role_from_hf
+
+本质上就是role_from_jsonl
+
+## 分别设置persona和role_name
+
+这个时候作为新人物考虑，默认没有RAG库，即Zero模式
+
+## 分别设置persona, role_name, texts
+
+这个时候会为texts再次抽取vectors
+
+## 分别设置persona, role_name, texts, vecs
+
+
+
+# 额外变量
+
+## max_input_token
+
+默认为1600，会根据这个来限制history的长度
+
+## user_name_in_message
+
+(这个功能在现在的预期核心代码中还没实现)
+
+默认为'default'， 当用户始终用同一个user_name和角色对话的时候，并不添加
+
+如果用户使用不同的role和chatbot聊天 user_name_in_message 会改为 'add' 并在每个message标记是谁说的
+
+（bot的也会添加）
+
+并且user_name替换为最后一个调用的user_name
+
+如果'not_add' 则永远不添加
+
+S MSG_U1 MSG_A MSG_U1 MSG_A
+
+当出现U2后
+
+S, U1:MSG_U1, A:MSG_A, U1:MSG_U1, A:MSG_A, U2:MSG_U2
+
+## token_counter
+
+tokenizer默认为gpt3.5的tiktoken，设置为None的时候，不进行任何的token长度限制
+
+## transfer_haruhi_2_zero
+
+(这个功能在现在的预期核心代码中还没实现)
+
+默认为true
+
+把原本ChatHaruhi的 角色: 「对话」的格式，去掉「」
+
+# Embedding
+
+中文考虑用bge_small
+
+Cross language考虑使用bce，相对还比较小， bge-m3太大了
+
+也就是ChatHaruhi类会有默认的embedding
+
+self.embedding = ChatHaruhi.bge_small
+
+对于输入的文本，我们会使用这个embedding来进行encode然后进行检索替换掉RAG的内容
+
+# 辅助接口
+
+## save_to_jsonl
+
+把一个角色保存成jsonl格式，方便上传hf
+
+
+# 预计的伪代码
+
+这里的核心就是去考虑ChatHaruhi下get_message函数的伪代码
+
+```python
+class ChatHaruhi:
+
+    def __init__( self ):
+        pass
+
+    def rag_retrieve( self, query_rags, rest_limit ):
+        # 返回一个rag_ids的列表
+        retrieved_ids = []
+        rag_ids = []
+
+        for query_rag in query_rags:
+            query = query_rag['query']
+            n = query_rag['n']
+            max_token = rest_limit
+            if rest_limit > query_rag['max_token'] and query_rag['max_token'] > 0:
+                max_token = query_rag['max_token']
+
+            rag_id = self.rag_retrieve( query, n, max_token, avoid_ids = retrieved_ids )
+            rag_ids.append( rag_id )
+            retrieved_ids += rag_id
+
+    def get_message(self, user, text):
+
+        query_token = self.token_counter( text )
+
+        # 首先获取需要多少个rag story
+        query_rags, persona_token = self.parse_persona( self.persona, text )
+        #每个query_rag需要饱含
+        # "n" 需要几个story
+        # "max_token" 最多允许多少个token，如果-1则不限制
+        # "query" 需要查询的内容
+        # "lid" 需要替换的行，这里直接进行行替换，忽视行的其他内容
+        
+        rest_limit = self.max_input_token - persona_token - query_token
+
+        rag_ids = self.rag_retrieve( query_rags, rest_limit )
+        
+        # 将rag_ids对应的故事 替换到persona中
+        augmented_persona = self.augment_persona( self.persona, rag_ids )
+
+        system_prompt = self.package_system_prompt( self.role_name, augmented_persona )
+
+        token_for_system = self.token_counter( system_prompt )
+
+        rest_limit = self.max_input_token - token_for_system - query_token
+
+        messages = [{"role":"system","content":system_prompt}]
+
+        messages = self.append_history_under_limit( messages, rest_limit )
+
+        messages.append({"role":"user",query})
+
+        return messages
+```

ChatHaruhi/SparkApi.py

@@ -0,0 +1,140 @@
+import _thread as thread
+import base64
+import datetime
+import hashlib
+import hmac
+import json
+from urllib.parse import urlparse
+import ssl
+from datetime import datetime
+from time import mktime
+from urllib.parse import urlencode
+from wsgiref.handlers import format_date_time
+
+import websocket  # 使用websocket_client
+answer = ""
+appid = None
+api_secret = None
+api_key = None
+
+class Ws_Param(object):
+    # 初始化
+    def __init__(self, APPID, APIKey, APISecret, Spark_url):
+        self.APPID = APPID
+        self.APIKey = APIKey
+        self.APISecret = APISecret
+        self.host = urlparse(Spark_url).netloc
+        self.path = urlparse(Spark_url).path
+        self.Spark_url = Spark_url
+
+    # 生成url
+    def create_url(self):
+        # 生成RFC1123格式的时间戳
+        now = datetime.now()
+        date = format_date_time(mktime(now.timetuple()))
+
+        # 拼接字符串
+        signature_origin = "host: " + self.host + "\n"
+        signature_origin += "date: " + date + "\n"
+        signature_origin += "GET " + self.path + " HTTP/1.1"
+
+        # 进行hmac-sha256进行加密
+        signature_sha = hmac.new(self.APISecret.encode('utf-8'), signature_origin.encode('utf-8'),
+                                 digestmod=hashlib.sha256).digest()
+
+        signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
+
+        authorization_origin = f'api_key="{self.APIKey}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
+
+        authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
+
+        # 将请求的鉴权参数组合为字典
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": self.host
+        }
+        # 拼接鉴权参数，生成url
+        url = self.Spark_url + '?' + urlencode(v)
+        # 此处打印出建立连接时候的url,参考本demo的时候可取消上方打印的注释，比对相同参数时生成的url与自己代码生成的url是否一致
+        return url
+
+
+# 收到websocket错误的处理
+def on_error(ws, error):
+    print("### error:", error)
+
+
+# 收到websocket关闭的处理
+def on_close(ws,one,two):
+    return
+    # print(" ")
+
+
+# 收到websocket连接建立的处理
+def on_open(ws):
+    thread.start_new_thread(run, (ws,))
+
+
+def run(ws, *args):
+    data = json.dumps(gen_params(appid=ws.appid, domain= ws.domain,question=ws.question))
+    ws.send(data)
+
+
+# 收到websocket消息的处理
+def on_message(ws, message):
+    # print(message)
+    data = json.loads(message)
+    code = data['header']['code']
+    if code != 0:
+        print(f'请求错误: {code}, {data}')
+        ws.close()
+    else:
+        choices = data["payload"]["choices"]
+        status = choices["status"]
+        content = choices["text"][0]["content"]
+        # print(content,end ="")
+        global answer
+        answer += content
+        # print(1)
+        if status == 2:
+            ws.close()
+
+
+def gen_params(appid, domain,question):
+    """
+    通过appid和用户的提问来生成请参数
+    """
+    data = {
+        "header": {
+            "app_id": appid,
+            "uid": "1234"
+        },
+        "parameter": {
+            "chat": {
+                "domain": domain,
+                "temperature": 0.5,
+                "max_tokens": 2048
+            }
+        },
+        "payload": {
+            "message": {
+                "text": question
+            }
+        }
+    }
+    return data
+
+
+def main(appid, api_key, api_secret, Spark_url,domain, question):
+    # print("星火:")
+    wsParam = Ws_Param(appid, api_key, api_secret, Spark_url)
+    websocket.enableTrace(False)
+    wsUrl = wsParam.create_url()
+    ws = websocket.WebSocketApp(wsUrl, on_message=on_message, on_error=on_error, on_close=on_close, on_open=on_open)
+    ws.appid = appid
+    ws.question = question
+    ws.domain = domain
+    ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
+
+

ChatHaruhi/__init__.py

@@ -0,0 +1,2 @@
+
+from .ChatHaruhi import ChatHaruhi

ChatHaruhi/embeddings.py

@@ -0,0 +1,270 @@
+import random
+
+# elif embedding == 'bge_en':
+#                 embed_name = 'bge_en_s15'
+#             elif embedding == 'bge_zh':
+#                 embed_name = 'bge_zh_s15'
+
+import torch
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+embedshortname2model_name = {
+    "bge_zh":"BAAI/bge-small-zh-v1.5",
+}
+
+embedname2columnname = {
+    "luotuo_openai":"luotuo_openai",
+    "openai":"luotuo_openai",
+    "bge_zh":"bge_zh_s15",
+    "bge_en":"bge_en_s15",
+    "bce":"bce_base",
+}
+
+# 这是用来调试的foo embedding
+
+def foo_embedding( text ):
+    # whatever text input , output a 2 dim 0-1 random vects
+    return [random.random(), random.random()]
+    
+# TODO: add bge-zh-small(or family)  BCE and openai embedding here 米唯实
+# ======== add bge_zh mmodel
+# by Weishi MI
+
+def foo_bge_zh_15( text ):
+    dim = 512
+    model_name = "BAAI/bge-small-zh-v1.5"
+    if isinstance(text, str):
+        text_list = [text]
+    else:
+        get_general_embeddings_safe(text, model_name)
+    
+    global _model_pool
+    global _tokenizer_pool
+
+    if model_name not in _model_pool:
+        from transformers import AutoTokenizer, AutoModel
+        _tokenizer_pool[model_name] = AutoTokenizer.from_pretrained(model_name)
+        _model_pool[model_name] = AutoModel.from_pretrained(model_name)
+
+    _model_pool[model_name].eval()
+
+    # Tokenize sentences
+    encoded_input = _tokenizer_pool[model_name](text_list, padding=True, truncation=True, return_tensors='pt', max_length = 512)
+
+    # Compute token embeddings
+    with torch.no_grad():
+        model_output = _model_pool[model_name](**encoded_input)
+        # Perform pooling. In this case, cls pooling.
+        sentence_embeddings = model_output[0][:, 0]
+
+    # normalize embeddings
+    sentence_embeddings = torch.nn.functional.normalize(sentence_embeddings, p=2, dim=1)
+    return sentence_embeddings.cpu().tolist()[0]
+    # return [random.random() for _ in range(dim)]
+
+def foo_bce( text ):
+    from transformers import AutoModel, AutoTokenizer
+    if isinstance(text, str):
+        text_list = [text]
+    
+    # init model and tokenizer
+    tokenizer = AutoTokenizer.from_pretrained('maidalun1020/bce-embedding-base_v1')
+    model = AutoModel.from_pretrained('maidalun1020/bce-embedding-base_v1')
+    
+    model.to(device)
+    
+    # get inputs
+    inputs = tokenizer(text_list, padding=True, truncation=True, max_length=512, return_tensors="pt")
+    inputs_on_device = {k: v.to(self.device) for k, v in inputs.items()}
+    
+    # get embeddings
+    outputs = model(**inputs_on_device, return_dict=True)
+    embeddings = outputs.last_hidden_state[:, 0]  # cls pooler
+    embeddings = embeddings / embeddings.norm(dim=1, keepdim=True)  # normalize
+    return embeddings
+def download_models():
+    print("正在下载Luotuo-Bert")
+    # Import our models. The package will take care of downloading the models automatically
+    model_args = Namespace(do_mlm=None, pooler_type="cls", temp=0.05, mlp_only_train=False,
+                           init_embeddings_model=None)
+    model = AutoModel.from_pretrained("silk-road/luotuo-bert-medium", trust_remote_code=True, model_args=model_args).to(
+        device)
+    print("Luotuo-Bert下载完毕")
+    return model
+
+def get_luotuo_model():
+    global _luotuo_model
+    if _luotuo_model is None:
+        _luotuo_model = download_models()
+    return _luotuo_model
+
+
+def luotuo_embedding(model, texts):
+    # Tokenize the texts_source
+    tokenizer = AutoTokenizer.from_pretrained("silk-road/luotuo-bert-medium")
+    inputs = tokenizer(texts, padding=True, truncation=False, return_tensors="pt")
+    inputs = inputs.to(device)
+    # Extract the embeddings
+    # Get the embeddings
+    with torch.no_grad():
+        embeddings = model(**inputs, output_hidden_states=True, return_dict=True, sent_emb=True).pooler_output
+    return embeddings
+
+def luotuo_en_embedding( texts ):
+    # this function implemented by Cheng
+    global _luotuo_model_en
+    global _luotuo_en_tokenizer
+
+    if _luotuo_model_en is None:
+        _luotuo_en_tokenizer = AutoTokenizer.from_pretrained("silk-road/luotuo-bert-en")
+        _luotuo_model_en = AutoModel.from_pretrained("silk-road/luotuo-bert-en").to(device)
+
+    if _luotuo_en_tokenizer is None:
+        _luotuo_en_tokenizer = AutoTokenizer.from_pretrained("silk-road/luotuo-bert-en")
+
+    inputs = _luotuo_en_tokenizer(texts, padding=True, truncation=False, return_tensors="pt")
+    inputs = inputs.to(device)
+
+    with torch.no_grad():
+        embeddings = _luotuo_model_en(**inputs, output_hidden_states=True, return_dict=True, sent_emb=True).pooler_output
+        
+    return embeddings
+
+
+def get_embedding_for_chinese(model, texts):
+    model = model.to(device)
+    # str or strList
+    texts = texts if isinstance(texts, list) else [texts]
+    # 截断
+    for i in range(len(texts)):
+        if len(texts[i]) > 510:
+            texts[i] = texts[i][:510]
+    if len(texts) >= 64:
+        embeddings = []
+        chunk_size = 64
+        for i in range(0, len(texts), chunk_size):
+            embeddings.append(luotuo_embedding(model, texts[i: i + chunk_size]))
+        return torch.cat(embeddings, dim=0)
+    else:
+        return luotuo_embedding(model, texts)
+
+
+def is_chinese_or_english(text):
+    # no longer use online openai api
+    return "chinese"
+
+    text = list(text)
+    is_chinese, is_english = 0, 0
+
+    for char in text:
+        # 判断字符的Unicode值是否在中文字符的Unicode范围内
+        if '\u4e00' <= char <= '\u9fa5':
+            is_chinese += 4
+        # 判断字符是否为英文字符（包括大小写字母和常见标点符号）
+        elif ('\u0041' <= char <= '\u005a') or ('\u0061' <= char <= '\u007a'):
+            is_english += 1
+    if is_chinese >= is_english:
+        return "chinese"
+    else:
+        return "english"
+
+
+def get_embedding_openai(text, model="text-embedding-ada-002"):
+    text = text.replace("\n", " ")
+    return client.embeddings.create(input = [text], model=model).data[0].embedding
+
+def get_embedding_for_english(text, model="text-embedding-ada-002"):
+    text = text.replace("\n", " ")
+    return client.embeddings.create(input = [text], model=model).data[0].embedding
+
+import os
+
+def foo_openai( text ):
+    # dim = 1536
+
+    openai_key = os.environ.get("OPENAI_API_KEY")
+
+    if isinstance(texts, list):
+        index = random.randint(0, len(texts) - 1)
+        if openai_key is None or is_chinese_or_english(texts[index]) == "chinese":
+            return [embed.cpu().tolist() for embed in get_embedding_for_chinese(get_luotuo_model(), texts)]
+        else:
+            return [get_embedding_for_english(text) for text in texts]
+    else:
+        if openai_key is None or is_chinese_or_english(texts) == "chinese":
+            return get_embedding_for_chinese(get_luotuo_model(), texts)[0].cpu().tolist()
+        else:
+            return get_embedding_for_english(texts)
+
+
+### BGE family
+
+
+# ======== add bge_zh mmodel
+# by Cheng Li
+# 这一次我们试图一次性去适配更多的模型
+import torch
+
+_model_pool = {}
+_tokenizer_pool = {}
+
+# BAAI/bge-small-zh-v1.5
+
+def get_general_embeddings( sentences , model_name = "BAAI/bge-small-zh-v1.5" ):
+
+    global _model_pool
+    global _tokenizer_pool
+
+    if model_name not in _model_pool:
+        from transformers import AutoTokenizer, AutoModel
+        _tokenizer_pool[model_name] = AutoTokenizer.from_pretrained(model_name)
+        _model_pool[model_name] = AutoModel.from_pretrained(model_name).to(device)
+
+    _model_pool[model_name].eval()
+
+    # Tokenize sentences
+    encoded_input = _tokenizer_pool[model_name](sentences, padding=True, truncation=True, return_tensors='pt', max_length = 512).to(device)
+
+    # Compute token embeddings
+    with torch.no_grad():
+        model_output = _model_pool[model_name](**encoded_input)
+        # Perform pooling. In this case, cls pooling.
+        sentence_embeddings = model_output[0][:, 0]
+
+    # normalize embeddings
+    sentence_embeddings = torch.nn.functional.normalize(sentence_embeddings, p=2, dim=1)
+    return sentence_embeddings.cpu().tolist()
+
+def get_general_embedding( text_or_texts , model_name = "BAAI/bge-small-zh-v1.5" ):
+    if isinstance(text_or_texts, str):
+        return get_general_embeddings([text_or_texts], model_name)[0]
+    else:
+        return get_general_embeddings_safe(text_or_texts, model_name)
+    
+general_batch_size = 16
+
+import math
+
+def get_general_embeddings_safe(sentences, model_name = "BAAI/bge-small-zh-v1.5"):
+    
+    embeddings = []
+    
+    num_batches = math.ceil(len(sentences) / general_batch_size)
+
+    from tqdm import tqdm
+    
+    for i in tqdm( range(num_batches) ):
+        # print("run bge with batch ", i)
+        start_index = i * general_batch_size
+        end_index = min(len(sentences), start_index + general_batch_size)
+        batch = sentences[start_index:end_index]
+        embs = get_general_embeddings(batch, model_name)
+        embeddings.extend(embs)
+        
+    return embeddings
+
+def get_bge_zh_embedding( text_or_texts ):
+    return get_general_embedding(text_or_texts, "BAAI/bge-small-zh-v1.5")
+

ChatHaruhi/novel_extract.py

@@ -0,0 +1,176 @@
+
+
+
+
+import re
+
+def extract_speaker(text):
+    # 使用正则表达式匹配文本开头的 "<name> :" 格式，并捕获冒号后面的内容
+    match = re.match(r'^([^:]+) :(.*)', text)
+    if match:
+        return (match.group(1), match.group(2).strip())  # 返回匹配到的name部分和冒号后面的内容作为元组
+    else:
+        return None, text  # 如果不匹配，返回None和原始文本
+
+
+def get_line_recall(query, line):
+    # 获得query中每个汉字在 line 中的recall
+    if not query or not line:
+        return 0
+    line_set = set(line)
+    return sum(char in line_set for char in query) / len(query)
+
+
+def get_max_recall_in_lines(query, lines):
+    recall_values = [(get_line_recall(query, line), i) for i, line in enumerate(lines)]
+    return max(recall_values, default=(-1, -1), key=lambda x: x[0])
+
+def extract_dialogues_from_response(text):
+    # Split the text into lines
+    lines = text.split('\n')
+
+    # Initialize an empty list to store the extracted dialogues
+    extracted_dialogues = []
+
+    valid_said_by = ["said by", "thought by", "described by", "from"]
+
+    # Iterate through each line
+    for line in lines:
+        # Split the line by '|' and strip whitespace from each part
+        parts = [part.strip() for part in line.split('|')]
+
+        # Check if the line has 4 parts and the third part is 'said by'
+        if len(parts) == 3:
+            # Extract the dialogue and speaker, and add to the list
+            if parts[2] == "speaker":
+                continue
+
+            if parts[1].strip().lower() not in valid_said_by:
+                continue
+
+            dialogue_dict = {
+                'dialogue': parts[0],
+                'speaker': parts[2],
+                "said_by": parts[1]
+            }
+            extracted_dialogues.append(dialogue_dict)
+
+    return extracted_dialogues
+
+
+def extract_dialogues_from_glm_response(text):
+    # Split the text into lines
+    lines = text.split('\n')
+
+    # Initialize an empty list to store the extracted dialogues
+    extracted_dialogues = []
+
+    valid_said_by = ["said by", "thought by", "described by", "from"]
+
+    # Iterate through each line
+    for line in lines:
+        # Split the line by '|' and strip whitespace from each part
+        parts = [part.strip() for part in line.split('|')]
+
+        # Check if the line has 4 parts and the third part is 'said by'
+        if len(parts) == 4:
+            # Extract the dialogue and speaker, and add to the list
+            if parts[3] == "speaker":
+                continue
+
+            if parts[2].strip().lower() not in valid_said_by:
+                continue
+
+            try:
+                id_num = int(parts[0])
+            except ValueError:
+                id_num = id
+
+            dialogue_dict = {
+                'id': id_num,
+                'dialogue': parts[1],
+                'speaker': parts[3],
+                "said_by": parts[2]
+            }
+            extracted_dialogues.append(dialogue_dict)
+
+    return extracted_dialogues
+
+
+def has_dialogue_sentences(text: str) -> int:
+    # 定义成对的引号
+    paired_quotes = [
+        ("“", "”"),
+        ("‘", "’"),
+        ("「", "」")
+    ]
+    # 定义符号列表（包括全角和半角的逗号和句号）
+    symbols = ['。', '!', '?', '*', '.', '？', '！', '"', '”', ',', '~', ')', '）', '…', ']', '♪','，']
+
+    # 检查成对引号内的内容
+    for start_quote, end_quote in paired_quotes:
+        start_index = text.find(start_quote)
+        while start_index != -1:
+            end_index = text.find(end_quote, start_index + 1)
+            if end_index != -1:
+                quote_content = text[start_index + 1:end_index]
+                # 检查引号内的内容是否符合条件
+                if any(symbol in quote_content for symbol in symbols) or len(quote_content) >= 10:
+                    return 2  # 成对引号内有符号或长度>=10
+                start_index = text.find(start_quote, end_index + 1)
+            else:
+                break
+
+    # 检查双引号'"'
+    double_quotes_indices = [i for i, char in enumerate(text) if char == '"']
+    if len(double_quotes_indices) % 2 == 0:  # 必须是偶数个双引号
+        for i in range(0, len(double_quotes_indices), 2):
+            start_index, end_index = double_quotes_indices[i], double_quotes_indices[i+1]
+            quote_content = text[start_index+1:end_index]
+            # 检查引号内的内容是否含有符号
+            if any(symbol in quote_content for symbol in symbols):
+                return 1  # 双引号内有符号
+
+    return 0  # 没有符合条件的对话型句子
+
+def replace_recalled_dialogue( raw_text, response_text ):
+    dialogues = extract_dialogues_from_response( response_text )
+
+    lines = raw_text.split("\n")
+
+    lines = [line.strip().strip("\u3000") for line in lines]
+
+    recall_flag = [ False for line in lines ]
+    line2ids = [ [] for line in lines ]
+
+    for id, dialogue in enumerate(dialogues):
+        dialogue_text = dialogue['dialogue']
+        remove_symbol_text = dialogue_text.replace("*","").replace('"',"")
+
+        recall, lid = get_max_recall_in_lines( remove_symbol_text, lines )
+
+        if recall > 0.3:
+            recall_flag[lid] = True
+            line2ids[lid].append(id)
+
+    new_text = ""
+
+    for lid, line in enumerate(lines):
+        if recall_flag[lid]:
+            if len(line2ids[lid]) == 1 and ("未知" in dialogues[0]['speaker'] or dialogues[0]['speaker'].strip() == ""):
+                new_text += line + "\n"
+                continue
+
+            for dia_id in line2ids[lid]:
+                speaker = dialogues[dia_id]['speaker']
+                dialogue = dialogues[dia_id]['dialogue']
+                dialogue = dialogue.replace('"',"").replace('“',"").replace('”',"")
+                new_text += speaker + " : " + dialogue + "\n"
+        else:
+            new_text += line + "\n"
+
+    return new_text.strip()
+
+
+
+            

ChatHaruhi/response_GLM_local.py

@@ -0,0 +1,121 @@
+import os
+from string import Template
+from typing import List, Dict
+
+import torch.cuda
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+aclient = None
+
+client = None
+tokenizer = None
+
+END_POINT = "https://hf-mirror.com"
+
+
+def init_client(model_name: str, verbose: bool) -> None:
+    """
+        初始化模型，通过可用的设备进行模型加载推理。
+
+        Params:
+            model_name (`str`)
+                HuggingFace中的模型项目名，例如"THUDM/chatglm3-6b"
+    """
+
+    # 将client设置为全局变量
+    global client
+    global tokenizer
+
+    # 判断 使用MPS、CUDA、CPU运行模型
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    elif torch.backends.mps.is_available():
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+
+    if verbose:
+        print("Using device: ", device)
+
+    # TODO 考虑支持deepspeed 进行多gpu推理，以及zero
+
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_name, trust_remote_code=True, local_files_only=True)
+        client = AutoModelForCausalLM.from_pretrained(
+            model_name, trust_remote_code=True, local_files_only=True)
+    except Exception:
+        if pretrained_model_download(model_name, verbose=verbose):
+            tokenizer = AutoTokenizer.from_pretrained(
+                model_name, trust_remote_code=True, local_files_only=True)
+            client = AutoModelForCausalLM.from_pretrained(
+                model_name, trust_remote_code=True, local_files_only=True)
+
+    client = client.to(device).eval()
+
+
+def pretrained_model_download(model_name_or_path: str, verbose: bool) -> bool:
+    """
+        使用huggingface_hub下载模型（model_name_or_path）。下载成功返回true，失败返回False。
+        Params: 
+            model_name_or_path (`str`): 模型的huggingface地址
+        Returns:
+            `bool` 是否下载成功
+    """
+    # TODO 使用hf镜像加速下载 未测试windows端
+
+    # 判断是否使用HF_transfer，默认不使用。
+    if os.getenv("HF_HUB_ENABLE_HF_TRANSFER") == 1:
+        try:
+            import hf_transfer
+        except ImportError:
+            print("Install hf_transfer.")
+            os.system("pip -q install hf_transfer")
+            import hf_transfer
+
+    # 尝试引入huggingface_hub
+    try:
+        import huggingface_hub
+    except ImportError:
+        print("Install huggingface_hub.")
+        os.system("pip -q install huggingface_hub")
+        import huggingface_hub
+
+    # 使用huggingface_hub下载模型。
+    try:
+        print(f"downloading {model_name_or_path}")
+        huggingface_hub.snapshot_download(
+            repo_id=model_name_or_path, endpoint=END_POINT, resume_download=True, local_dir_use_symlinks=False)
+    except Exception as e:
+        raise e
+
+    return True
+
+
+def message2query(messages: List[Dict[str, str]]) -> str:
+    # [{'role': 'user', 'content': '老师: 同学请自我介绍一下'}]
+    # <|system|>
+    # You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.
+    # <|user|>
+    # Hello
+    # <|assistant|>
+    # Hello, I'm ChatGLM3. What can I assist you today?
+    template = Template("<|$role|>\n$content\n")
+
+    return "".join([template.substitute(message) for message in messages])
+
+
+def get_response(message, model_name: str = "THUDM/chatglm3-6b", verbose: bool = False):
+    global client
+    global tokenizer
+
+    if client is None:
+        init_client(model_name, verbose=verbose)
+
+    if verbose:
+        print(message)
+        print(message2query(message))
+
+    response, history = client.chat(tokenizer, message2query(message))
+
+    return response

ChatHaruhi/response_GLM_lora.py

@@ -0,0 +1,133 @@
+import os
+from string import Template
+from typing import List, Dict
+
+import torch.cuda
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import AutoPeftModelForCausalLM
+
+
+aclient = None
+
+client = None
+tokenizer = None
+
+END_POINT = "https://hf-mirror.com"
+
+
+def init_client(model_name: str, verbose: bool) -> None:
+    """
+        初始化模型，通过可用的设备进行模型加载推理。
+
+        Params:
+            model_name (`str`)
+                HuggingFace中的模型项目名，例如"THUDM/chatglm3-6b"
+    """
+
+    # 将client设置为全局变量
+    global client
+    global tokenizer
+
+    # 判断 使用MPS、CUDA、CPU运行模型
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    elif torch.backends.mps.is_available():
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+
+    if verbose:
+        print("Using device: ", device)
+
+    # TODO 上传模型后，更改为从huggingface获取模型
+    client = AutoPeftModelForCausalLM.from_pretrained(
+        model_name, trust_remote_code=True)
+    tokenizer_dir = client.peft_config['default'].base_model_name_or_path
+    if verbose:
+        print(tokenizer_dir)
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_dir, trust_remote_code=True)
+
+    # try:
+    #     tokenizer = AutoTokenizer.from_pretrained(
+    #         model_name, trust_remote_code=True, local_files_only=True)
+    #     client = AutoModelForCausalLM.from_pretrained(
+    #         model_name, trust_remote_code=True, local_files_only=True)
+    # except Exception:
+    #     if pretrained_model_download(model_name, verbose=verbose):
+    #         tokenizer = AutoTokenizer.from_pretrained(
+    #             model_name, trust_remote_code=True, local_files_only=True)
+    #         client = AutoModelForCausalLM.from_pretrained(
+    #             model_name, trust_remote_code=True, local_files_only=True)
+
+    # client = client.to(device).eval()
+    client = client.eval()
+
+
+def pretrained_model_download(model_name_or_path: str, verbose: bool) -> bool:
+    """
+        使用huggingface_hub下载模型（model_name_or_path）。下载成功返回true，失败返回False。
+        Params: 
+            model_name_or_path (`str`): 模型的huggingface地址
+        Returns:
+            `bool` 是否下载成功
+    """
+    # TODO 使用hf镜像加速下载 未测试windows端
+
+    # 判断是否使用HF_transfer，默认不使用。
+    if os.getenv("HF_HUB_ENABLE_HF_TRANSFER") == 1:
+        try:
+            import hf_transfer
+        except ImportError:
+            print("Install hf_transfer.")
+            os.system("pip -q install hf_transfer")
+            import hf_transfer
+
+    # 尝试引入huggingface_hub
+    try:
+        import huggingface_hub
+    except ImportError:
+        print("Install huggingface_hub.")
+        os.system("pip -q install huggingface_hub")
+        import huggingface_hub
+
+    # 使用huggingface_hub下载模型。
+    try:
+        print(f"downloading {model_name_or_path}")
+        huggingface_hub.snapshot_download(
+            repo_id=model_name_or_path, endpoint=END_POINT, resume_download=True, local_dir_use_symlinks=False)
+    except Exception as e:
+        raise e
+
+    return True
+
+
+def message2query(messages: List[Dict[str, str]]) -> str:
+    # [{'role': 'user', 'content': '老师: 同学请自我介绍一下'}]
+    # <|system|>
+    # You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.
+    # <|user|>
+    # Hello
+    # <|assistant|>
+    # Hello, I'm ChatGLM3. What can I assist you today?
+    template = Template("<|$role|>\n$content\n")
+
+    return "".join([template.substitute(message) for message in messages])
+
+
+def get_response(message, model_name: str = "/workspace/jyh/Zero-Haruhi/checkpoint-1500", verbose: bool = True):
+    global client
+    global tokenizer
+
+    if client is None:
+        init_client(model_name, verbose=verbose)
+
+    if verbose:
+        print(message)
+        print(message2query(message))
+
+    response, history = client.chat(tokenizer, message2query(message))
+    if verbose:
+        print((response, history))
+
+    return response

ChatHaruhi/response_erniebot.py

@@ -0,0 +1,90 @@
+import erniebot
+
+aclient = None
+
+client = None
+
+import os
+
+def normalize2uaua_ernie( message, if_replace_system = False ):
+    new_message = []
+    last_role = ""
+
+    for msg in message:
+        role = msg["role"]
+        if if_replace_system and role == "system":
+            role = "user"
+            msg["role"] = role
+        
+        if last_role == role:
+            new_message[-1]["content"] = new_message[-1]["content"] + "\n" + msg["content"]
+        else:
+            last_role = role
+            new_message.append( msg )
+
+    return new_message
+
+def init_client():
+    
+    # 将client设置为全局变量
+    global client
+    
+    # 将ERNIE_ACCESS_TOKEN作为参数值传递给OS
+    api_key = os.getenv("ERNIE_ACCESS_TOKEN")
+    if api_key is None:
+        raise ValueError("环境变量'ERNIE_ACCESS_TOKEN'未设置，请确保已经定义了API密钥")
+    erniebot.api_type = "aistudio"
+    erniebot.access_token = api_key
+    client = erniebot
+
+def get_response( message, model_name = "ernie-4.0" ):
+    if client is None:
+        init_client()
+    
+    message_ua = normalize2uaua_ernie(message, if_replace_system = True)
+    # print(message_ua)
+    response = client.ChatCompletion.create(\
+        model=model_name,\
+        messages = message_ua, \
+        temperature = 0.1 )
+    return response.get_result()
+
+import json
+import asyncio
+from erniebot_agent.chat_models import ERNIEBot
+from erniebot_agent.memory import HumanMessage, AIMessage, SystemMessage, FunctionMessage
+
+def init_aclient(model="ernie-4.0"):
+    
+    # 将aclient设置为全局变量
+    global aclient
+
+    api_key = os.getenv("ERNIE_ACCESS_TOKEN")
+    if api_key is None:
+        raise ValueError("环境变量'ERNIE_ACCESS_TOKEN'未设置。请确保已经定义了API密钥。")
+    os.environ["EB_AGENT_ACCESS_TOKEN"] = api_key
+    aclient = ERNIEBot(model=model)  # 创建模型
+
+
+
+async def async_get_response( message, model="ernie-4.0" ):
+    if aclient is None:
+        init_aclient(model=model)
+    
+    messages = []
+    system_message = None
+    message_ua = normalize2uaua_ernie(message, if_replace_system = False)
+    print(message_ua)
+    for item in message_ua:
+        if item["role"] == "user":
+            messages.append(HumanMessage(item["content"]))
+        elif item["role"] == "system":
+            system_message = SystemMessage(item["content"])
+        else:
+            messages.append(AIMessage(item["content"])) 
+    if system_message:
+        ai_message = await aclient.chat(messages=messages, temperature = 0.1)
+    else:
+        ai_message = await aclient.chat(messages=messages, system=system_message.content, temperature = 0.1)  # 调用模型chat接口，非流式返回
+
+    return ai_message.content

ChatHaruhi/response_openai.py

@@ -0,0 +1,65 @@
+import openai
+
+aclient = None
+
+client = None
+
+import os
+from openai import OpenAI
+
+def init_client():
+    # 将client设置为全局变量，以便在其他函数中使用
+    global client
+
+    # 检查是否存在API_KEY环境变量
+    api_key = os.getenv("OPENAI_API_KEY")
+    if api_key is None:
+        raise ValueError("环境变量'OPENAI_API_KEY'未设置。请确保已经定义了API密钥。")
+    
+    # 检查是否存在API_BASE环境变量，并据此设置base_url参数
+    api_base = os.getenv("OPENAI_API_BASE")
+    if api_base:
+        client = OpenAI(base_url=api_base, api_key=api_key)
+    else:
+        client = OpenAI(api_key=api_key)
+
+    
+
+def get_response( message ):
+    if client is None:
+        init_client()
+    response = client.chat.completions.create(\
+        model="gpt-3.5-turbo",\
+        messages = message, \
+        max_tokens = 300, \
+        temperature = 0.1 )
+    return response.choices[0].message.content
+
+from openai import AsyncOpenAI
+
+def init_aclient():
+    # 将aclient设置为全局变量，以便在其他函数中使用
+    global aclient
+
+    # 检查是否存在API_KEY环境变量
+    api_key = os.getenv("OPENAI_API_KEY")
+    if api_key is None:
+        raise ValueError("环境变量'OPENAI_API_KEY'未设置。请确保已经定义了API密钥。")
+    
+    # 检查是否存在API_BASE环境变量，并据此设置base_url参数
+    api_base = os.getenv("OPENAI_API_BASE")
+    if api_base:
+        aclient = AsyncOpenAI(base_url=api_base, api_key=api_key)
+    else:
+        aclient = AsyncOpenAI(api_key=api_key)
+
+async def async_get_response( message ):
+    if aclient is None:
+        init_aclient()
+    response = await aclient.chat.completions.create(\
+        model="gpt-3.5-turbo",\
+        messages = message, \
+        max_tokens = 300, \
+        temperature = 0.1 )
+    return response.choices[0].message.content
+    

ChatHaruhi/response_spark.py

@@ -0,0 +1,51 @@
+from . import SparkApi
+
+aclient = None
+
+client = None
+
+import os
+
+def init_client():
+
+    # 将client设置为全局变量
+    global client
+
+    # 将ERNIE_ACCESS_TOKEN作为参数值传递给OS
+    appid = os.getenv("SPARK_APPID")
+    api_secret = os.getenv("SPARK_API_SECRET")
+    api_key = os.getenv("SPARK_API_KEY")
+    if appid is None:
+        raise ValueError("环境变量'SPARK_APPID'未设置，请确保已经定义了API密钥")
+    if api_secret is None:
+        raise ValueError("环境变量'SPARK_API_SECRET'未设置，请确保已经定义了API密钥")
+    if api_key is None:
+        raise ValueError("环境变量'SPARK_API_KEY'未设置，请确保已经定义了API密钥")
+    SparkApi.appid = appid
+    SparkApi.api_secret = api_secret
+    SparkApi.api_key = api_key
+    client = SparkApi
+
+def get_response(message, model_name = "Spark3.5"):
+    if client is None:
+        init_client()
+
+    if model_name == "Spark2.0":
+        domain = "generalv2"    # v2.0版本
+        Spark_url = "ws://spark-api.xf-yun.com/v2.1/chat"  # v2.0环境的地址
+    elif model_name == "Spark1.5":
+        domain = "general"   # v1.5版本
+        Spark_url = "ws://spark-api.xf-yun.com/v1.1/chat"  # v1.5环境的地址
+    elif model_name == "Spark3.0":
+        domain = "generalv3"   # v3.0版本
+        Spark_url = "ws://spark-api.xf-yun.com/v3.1/chat"  # v3.0环境的地址
+    elif model_name == "Spark3.5":
+        domain = "generalv3.5"   # v3.5版本
+        Spark_url = "ws://spark-api.xf-yun.com/v3.5/chat"  # v3.5环境的地址
+    else:
+        raise Exception("Unknown Spark model")
+    # print(message_ua)
+    client.answer = ""
+    client.main(client.appid,client.api_key,client.api_secret,Spark_url,domain,message)
+    return client.answer
+

ChatHaruhi/response_zhipu.py

@@ -0,0 +1,54 @@
+import zhipuai
+
+aclient = None
+
+client = None
+
+import os
+from zhipuai import ZhipuAI
+
+def init_client():
+    
+    # 将client设置为全局变量
+    global client
+    
+    # 将ZHIPUAI_API_KEY作为参数值传递给OS
+    api_key = os.getenv("ZHIPUAI_API_KEY")
+    if api_key is None:
+        raise ValueError("环境变量'ZHIPUAI_API_KEY'未设置，请确保已经定义了API密钥")
+    
+    client = ZhipuAI(api_key=api_key)
+
+ 
+def init_aclient():
+    
+    # 将aclient设置为全局变量
+    global aclient
+
+    # 将ZHIPUAI_API_KEY作为参数值传递给OS
+    api_key = os.getenv("ZHIPUAI_API_KEY")
+    if api_key is None:
+        raise ValueError("环境变量'ZHIPUAI_API_KEY'未设置，请确保已经定义了API密钥")
+
+def get_response( message, model_name = "glm-3-turbo" ):
+    if client is None:
+        init_client()
+    response = client.chat.completions.create(\
+        model=model_name,\
+        messages = message, \
+        max_tokens = 300, \
+        temperature = 0.1 )
+    return response.choices[0].message.content
+
+
+
+
+
+
+
+
+
+    
+
+
+    

ChatHaruhi/sugar_map.py

@@ -0,0 +1,30 @@
+sugar_role_names = {'汤师爷': 'tangshiye', 'tangshiye': 'tangshiye', 'Tangshiye': 'tangshiye', 
+                     '慕容复': 'murongfu', 'murongfu': 'murongfu', 'Murongfu': 'murongfu', 
+                     '李云龙': 'liyunlong', 'liyunlong': 'liyunlong', 'Liyunlong': 'liyunlong', 
+                     'Luna': 'Luna', '王多鱼': 'wangduoyu', 'wangduoyu': 'wangduoyu', 
+                     'Wangduoyu': 'wangduoyu', 'Ron': 'Ron', '鸠摩智': 'jiumozhi', 
+                     'jiumozhi': 'jiumozhi', 'Jiumozhi': 'jiumozhi', 'Snape': 'Snape', 
+                     '凉宫春日': 'haruhi', 'haruhi': 'haruhi', 'Haruhi': 'haruhi', 
+                     'Malfoy': 'Malfoy', '虚竹': 'xuzhu', 'xuzhu': 'xuzhu', 
+                     'Xuzhu': 'xuzhu', '萧峰': 'xiaofeng', 
+                     'xiaofeng': 'xiaofeng', 'Xiaofeng': 'xiaofeng', '段誉': 'duanyu', 
+                     'duanyu': 'duanyu', 'Duanyu': 'duanyu', 'Hermione': 'Hermione', 
+                     'Dumbledore': 'Dumbledore', '王语嫣': 'wangyuyan', 'wangyuyan': 
+                     'wangyuyan', 'Wangyuyan': 'wangyuyan', 'Harry': 'Harry', 
+                     'McGonagall': 'McGonagall', '白展堂': 'baizhantang', 
+                     'baizhantang': 'baizhantang', 'Baizhantang': 'baizhantang', 
+                     '佟湘玉': 'tongxiangyu', 'tongxiangyu': 'tongxiangyu', 
+                     'Tongxiangyu': 'tongxiangyu', '郭芙蓉': 'guofurong', 
+                     'guofurong': 'guofurong', 'Guofurong': 'guofurong', '流浪者': 'wanderer', 
+                     'wanderer': 'wanderer', 'Wanderer': 'wanderer', '钟离': 'zhongli', 
+                     'zhongli': 'zhongli', 'Zhongli': 'zhongli', '胡桃': 'hutao', 'hutao': 'hutao', 
+                     'Hutao': 'hutao', 'Sheldon': 'Sheldon', 'Raj': 'Raj', 
+                     'Penny': 'Penny', '韦小宝': 'weixiaobao', 'weixiaobao': 'weixiaobao', 
+                     'Weixiaobao': 'weixiaobao', '乔峰': 'qiaofeng', 'qiaofeng': 'qiaofeng', 
+                     'Qiaofeng': 'qiaofeng', '神里绫华': 'ayaka', 'ayaka': 'ayaka', 
+                     'Ayaka': 'ayaka', '雷电将军': 'raidenShogun', 'raidenShogun': 'raidenShogun', 
+                     'RaidenShogun': 'raidenShogun', '于谦': 'yuqian', 'yuqian': 'yuqian', 
+                     'Yuqian': 'yuqian', 'Professor McGonagall': 'McGonagall', 
+                     'Professor Dumbledore': 'Dumbledore'}
+
+enname2zhname = {'tangshiye': '汤师爷', 'murongfu': '慕容复', 'liyunlong': '李云龙', 'Luna': 'Luna', 'wangduoyu': '王多鱼', 'Ron': 'Ron', 'jiumozhi': '鸠摩智', 'Snape': 'Snape', 'haruhi': '凉宫春日', 'Malfoy': 'Malfoy', 'xuzhu': '虚竹', 'xiaofeng': '萧峰', 'duanyu': '段誉', 'Hermione': 'Hermione', 'Dumbledore': 'Dumbledore', 'wangyuyan': '王语嫣', 'Harry': 'Harry', 'McGonagall': 'McGonagall', 'baizhantang': '白展堂', 'tongxiangyu': '佟湘玉', 'guofurong': '郭芙蓉', 'wanderer': '流浪者', 'zhongli': '钟离', 'hutao': '胡桃', 'Sheldon': 'Sheldon', 'Raj': 'Raj', 'Penny': 'Penny', 'weixiaobao': '韦小宝', 'qiaofeng': '乔峰', 'ayaka': '神里绫华', 'raidenShogun': '雷电将军', 'yuqian': '于谦'}

ChatHaruhi/utils.py

@@ -0,0 +1,89 @@
+import tiktoken
+
+_enc_model = None
+
+def normalize2uaua( message, if_replace_system = False ):
+    new_message = []
+    last_role = ""
+
+    for msg in message:
+        role = msg["role"]
+        if if_replace_system and role == "system":
+            role = "user"
+        
+        if last_role == role:
+            new_message[-1]["content"] = new_message[-1]["content"] + "\n" + msg["content"]
+        else:
+            last_role = role
+            new_message.append( msg )
+
+    return new_message
+
+def tiktoken_counter( text ):
+    global _enc_model
+
+    if _enc_model is None:
+        _enc_model = tiktoken.get_encoding("cl100k_base")
+
+    return len(_enc_model.encode(text))
+
+
+def string_to_base64(text):
+    import base64
+    byte_array = b''
+    for char in text:
+        num_bytes = char.encode('utf-8')
+        byte_array += num_bytes
+
+    base64_data = base64.b64encode(byte_array)
+    return base64_data.decode('utf-8')
+
+def base64_to_string(base64_data):
+    import base64
+    byte_array = base64.b64decode(base64_data)
+    text = byte_array.decode('utf-8')
+    return text
+
+
+def float_array_to_base64(float_arr):
+    import struct
+    import base64
+    byte_array = b''
+    
+    for f in float_arr:
+        # 将每个浮点数打包为4字节
+        num_bytes = struct.pack('!f', f)  
+        byte_array += num_bytes
+    
+    # 将字节数组进行base64编码    
+    base64_data = base64.b64encode(byte_array)
+    
+    return base64_data.decode('utf-8')
+
+def base64_to_float_array(base64_data):
+    import struct
+    import base64
+    byte_array = base64.b64decode(base64_data)
+    
+    float_array = []
+    
+    # 每 4 个字节解析为一个浮点数
+    for i in range(0, len(byte_array), 4):
+        num = struct.unpack('!f', byte_array[i:i+4])[0] 
+        float_array.append(num)
+
+    return float_array
+
+def load_datas_from_jsonl( file_path ):
+    import json
+    datas = []
+    with open(file_path, 'r', encoding = 'utf-8') as f:
+        for line in f:
+            datas.append(json.loads(line))
+    return datas
+
+def save_datas_to_jsonl( file_path, datas ):
+    import json
+    with open(file_path, 'w', encoding = 'utf-8') as f:
+        for data in datas:
+            f.write(json.dumps(data, ensure_ascii=False) + '\n')