update

README.md

@@ -10,4 +10,12 @@ pinned: false
 license: apache-2.0
 ---
 
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+
+
+## 安装问题
+
+直接从源码安装，推理速度较慢，因此加入以下参数。
+```sh
+pip install git+https://github.com/abetlen/llama-cpp-python.git -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS"
+```

app.py

@@ -36,8 +36,9 @@ Essentially, it is a form of model compression.
 
 with gr.Blocks() as demo:
     # Knowledge Distillation through Self Chatting
-    #
-    gr.HTML("""<h1 align="center">Distilling the Knowledge from LLM through Self Chatting</h1>""")
+    # Distilling the Knowledge from LLM through Self Chatting
+    # Generating Synthetic Data through Self Chat
+    gr.HTML("""<h1 align="center">Generating Synthetic Data through Self Chat</h1>""")
     with gr.Row():
         with gr.Column(scale=5):
             system = gr.Dropdown(
@@ -53,6 +54,8 @@ with gr.Blocks() as demo:
                                  show_share_button=True,
                                  avatar_images=("assets/man.png", "assets/bot.png"))
 
+            gr.Textbox("For faster inference, you can build locally with  ")
+            # ss
             with gradio.Tab("Self Chat"):
                 input_text_1 = gr.Textbox(show_label=False, placeholder="...", lines=10, visible=False)
                 generate_btn = gr.Button("🤔️ Self-Chat", variant="primary")
@@ -66,6 +69,7 @@ with gr.Blocks() as demo:
                     "It is based on user simulator and response generator.",
                     visible=True)
 
+            # 也叫 chat-assistant,
             with gradio.Tab("Response Generator"):
                 with gr.Row():
                     input_text_2 = gr.Textbox(show_label=False, placeholder="Please type your input", scale=7)
@@ -76,6 +80,7 @@ with gr.Blocks() as demo:
                     clear_btn_2 = gr.Button("🗑️  Clear", variant="secondary", size="sm", )  # 🧹 Clear History (清除历史)
                 gr.Markdown("Response simulator is the most commonly used chatbot.")
 
+            #
             with gradio.Tab("User Simulator"):
                 with gr.Row():
                     input_text_3 = gr.Textbox(show_label=False, placeholder="Please type your response", scale=7)
@@ -96,8 +101,8 @@ with gr.Blocks() as demo:
                 # visible=False
             )
             with gr.Accordion(label="Parameters", open=True):
-                slider_max_tokens = gr.Slider(minimum=1, maximum=config.MAX_SEQUENCE_LENGTH,
-                                              value=config.DEFAULT_MAX_TOKENS, step=1, label="Max tokens")
+                slider_max_new_tokens = gr.Slider(minimum=1, maximum=4096,
+                                                  value=config.DEFAULT_MAX_NEW_TOKENS, step=1, label="Max tokens")
                 slider_temperature = gr.Slider(minimum=0.1, maximum=10.0,
                                                value=config.DEFAULT_TEMPERATURE, step=0.1, label="Temperature",
                                                info="Larger temperature increase the randomness")
@@ -136,7 +141,7 @@ with gr.Blocks() as demo:
         .then(generate, [chatbot, history], outputs=[chatbot, history],
               show_progress="full")
     undo_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
-    clear_btn_2.click(reset_state, inputs=[system], outputs=[chatbot, history])\
+    clear_btn_2.click(reset_state, inputs=[system], outputs=[chatbot, history]) \
         .then(reset_user_input, outputs=[input_text_2])
 
     ######## tab3
@@ -147,12 +152,10 @@ with gr.Blocks() as demo:
         .then(generate, [chatbot, history], outputs=[chatbot, history],
               show_progress="full")
     undo_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
-    clear_btn_3.click(reset_state, inputs=[system], outputs=[chatbot, history])\
+    clear_btn_3.click(reset_state, inputs=[system], outputs=[chatbot, history]) \
         .then(reset_user_input, outputs=[input_text_3])
 
-
-
-    slider_max_tokens.change(set_max_tokens, inputs=[slider_max_tokens])
+    slider_max_new_tokens.change(set_max_new_tokens, inputs=[slider_max_new_tokens])
     slider_temperature.change(set_temperature, inputs=[slider_temperature])
     slider_top_p.change(set_top_p, inputs=[slider_top_p])
     slider_top_k.change(set_top_k, inputs=[slider_top_k])

app_util.py

@@ -112,8 +112,8 @@ def reset_state(system):
     return [], [{"role": "system", "content": system}]
 
 
-def set_max_tokens(max_tokens):
-    bot.generation_kwargs["max_tokens"] = max_tokens
+def set_max_new_tokens(max_new_tokens):
+    bot.generation_kwargs["max_tokens"] = max_new_tokens
 
 def set_temperature(temperature):
     bot.generation_kwargs["temperature"] = temperature

config.py

@@ -1,9 +1,8 @@
 
 
-MAX_SEQUENCE_LENGTH = 2048  # max_seq_len
+MAX_SEQUENCE_LENGTH = 32768  # max_seq_len
 
-DEFAULT_MAX_TOKENS = 128
-# DEFAULT_MAX_NEW_TOKENS = None
+DEFAULT_MAX_NEW_TOKENS = 128
 DEFAULT_TOP_K = 100
 DEFAULT_TOP_P = 0.95
 DEFAULT_TEMPERATURE = 5

models/cpp_qwen2.py

@@ -1,6 +1,4 @@
 """
-
-
 ## convert to gguf
 
 python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/
@@ -15,49 +13,45 @@ python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Ins
 
 
 **重庆GPU服务器，cache为空 **
-llama_print_timings:        load time =    1711.48 ms
-llama_print_timings:      sample time =     214.87 ms /   122 runs   (    1.76 ms per token,   567.78 tokens per second)
-llama_print_timings: prompt eval time =     892.14 ms /     5 tokens (  178.43 ms per token,     5.60 tokens per second)
-llama_print_timings:        eval time =    4277.26 ms /   121 runs   (   35.35 ms per token,    28.29 tokens per second)
-llama_print_timings:       total time =    8351.28 ms /   126 tokens
-
-llama_print_timings:        load time =    1711.48 ms
-llama_print_timings:      sample time =      45.11 ms /    25 runs   (    1.80 ms per token,   554.24 tokens per second)
-llama_print_timings: prompt eval time =    1059.46 ms /     5 tokens (  211.89 ms per token,     4.72 tokens per second)
-llama_print_timings:        eval time =     843.71 ms /    24 runs   (   35.15 ms per token,    28.45 tokens per second)
-llama_print_timings:       total time =    2501.50 ms /    29 tokens
-
-
-llama_print_timings:        load time =    1711.48 ms
-llama_print_timings:      sample time =     227.75 ms /   125 runs   (    1.82 ms per token,   548.85 tokens per second)
-llama_print_timings: prompt eval time =    2056.86 ms /     5 tokens (  411.37 ms per token,     2.43 tokens per second)
-llama_print_timings:        eval time =    4657.86 ms /   124 runs   (   37.56 ms per token,    26.62 tokens per second)
-llama_print_timings:       total time =    9532.50 ms /   129 tokens
-
 llama_print_timings:        load time =    1711.48 ms
 llama_print_timings:      sample time =      73.89 ms /    41 runs   (    1.80 ms per token,   554.84 tokens per second)
-llama_print_timings: prompt eval time =    2621.25 ms /     5 tokens (  524.25 ms per token,     1.91 tokens per second)   # 0.5秒/token
+llama_print_timings: prompt eval time =    2621.25 ms /     5 tokens (  524.25 ms per token,     1.91 tokens per second)   # 0.2-0.5秒/token
 llama_print_timings:        eval time =    1430.91 ms /    40 runs   (   35.77 ms per token,    27.95 tokens per second)
 llama_print_timings:       total time =    4848.09 ms /    45 tokens
 
-**hf-space，cache为空 ** -----------
+llama_print_timings:        load time =    1939.72 ms
+llama_print_timings:      sample time =     286.69 ms /   170 runs   (    1.69 ms per token,   592.99 tokens per second)
+llama_print_timings: prompt eval time =       0.00 ms /     0 tokens (    -nan ms per token,     -nan tokens per second)  # warmup后，加速明显。
+llama_print_timings:        eval time =    5737.50 ms /   170 runs   (   33.75 ms per token,    29.63 tokens per second)
+llama_print_timings:       total time =    8219.82 ms /   170 tokens
+
+
+**hf-space，cache为空 （关闭GGML_BLAS） ** -----------
 llama_print_timings:        load time =   28230.06 ms
-llama_print_timings:      sample time =     147.58 ms /     8 runs   (   18.45 ms per token,    54.21 tokens per second)
-llama_print_timings: prompt eval time =   28864.82 ms /     5 tokens ( 5772.96 ms per token,     0.17 tokens per second)   # 5.7秒/token
+llama_print_timings:      sample time =     147.58 ms /     8 runs   (   18.45 ms per token,    54.21 tokens per second)   # 18ms/token
+llama_print_timings: prompt eval time =   28864.82 ms /     5 tokens ( 5772.96 ms per token,     0.17 tokens per second)   # 5.7s/token
 llama_print_timings:        eval time =    1557.94 ms /     7 runs   (  222.56 ms per token,     4.49 tokens per second)
 llama_print_timings:       total time =   30753.48 ms /    12 tokens
 
-llama_print_timings:        load time =   28230.06 ms
-llama_print_timings:      sample time =      74.34 ms /    61 runs   (    1.22 ms per token,   820.52 tokens per second)
-llama_print_timings: prompt eval time =   28821.26 ms /     9 tokens ( 3202.36 ms per token,     0.31 tokens per second)
-llama_print_timings:        eval time =   21634.71 ms /    60 runs   (  360.58 ms per token,     2.77 tokens per second)
-llama_print_timings:       total time =   51255.55 ms /    69 tokens
 
-llama_print_timings:        load time =   28230.06 ms
-llama_print_timings:      sample time =      98.03 ms /    68 runs   (    1.44 ms per token,   693.66 tokens per second)
-llama_print_timings: prompt eval time =   27749.35 ms /     5 tokens ( 5549.87 ms per token,     0.18 tokens per second)
-llama_print_timings:        eval time =   26998.58 ms /    67 runs   (  402.96 ms per token,     2.48 tokens per second)
-llama_print_timings:       total time =   56335.37 ms /    72 tokens
+**hf-space，cache为空 （开启GGML_BLAS）** -----------
+llama_print_timings:        load time =   27347.29 ms
+llama_print_timings:      sample time =      82.53 ms /    26 runs   (    3.17 ms per token,   315.05 tokens per second)   # 3ms/token
+llama_print_timings: prompt eval time =   28855.64 ms /     9 tokens ( 3206.18 ms per token,     0.31 tokens per second)   # 3s/token
+llama_print_timings:        eval time =    9810.01 ms /    25 runs   (  392.40 ms per token,     2.55 tokens per second)
+llama_print_timings:       total time =   39073.77 ms /    34 tokens
+
+llama_print_timings:        load time =   27347.29 ms
+llama_print_timings:      sample time =     272.12 ms /    96 runs   (    2.83 ms per token,   352.79 tokens per second)   # 2.8ms/token
+llama_print_timings: prompt eval time =       0.00 ms /     0 tokens (    -nan ms per token,     -nan tokens per second)
+llama_print_timings:        eval time =   19974.85 ms /    96 runs   (  208.07 ms per token,     4.81 tokens per second)
+llama_print_timings:       total time =   22517.08 ms /    96 tokens
+
+
+## TODO:
+
+- 解决warmup慢的问题
+- 支持cache，并提前对所有预设system进行cache。
 
 ## reference
 
@@ -75,7 +69,6 @@ import os
 
 from models.base_model import Simulator
 import llama_cpp
-# import llama_cpp.llama_tokenizer
 from transformers import AutoTokenizer
 from utils.logging_util import logger
 import config
@@ -121,7 +114,7 @@ class Qwen2Simulator(Simulator):
             temperature=config.DEFAULT_TEMPERATURE,
             top_p=config.DEFAULT_TOP_P,
             top_k=config.DEFAULT_TOP_K,
-            max_tokens=config.DEFAULT_MAX_TOKENS,
+            max_tokens=config.DEFAULT_MAX_NEW_TOKENS,
             repeat_penalty=1.1,
             # qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|>
             stop=self.stop_words,
@@ -131,12 +124,9 @@ class Qwen2Simulator(Simulator):
         self.assistant_start_tokens = self.tokenize("<|im_start|>assistant\n")
         # self.llm.generate  .set_cache   .last_n_tokens_size  .reset  .ctx ._ctx
 
-
-        self.cache_size=10
-
-        cache = llama_cpp.LlamaRAMCache(capacity_bytes=self.cache_size)
-
-        # self.llm.set_cache()
+        # cache = llama_cpp.LlamaDiskCache(capacity_bytes=cache_size)
+        cache = llama_cpp.LlamaRAMCache(capacity_bytes=2 << 30)
+        self.llm.set_cache(cache)
 
     def tokenize(self, text):
         return self.llm.tokenize(text.encode("utf-8"))
@@ -195,7 +185,8 @@ class Qwen2Simulator(Simulator):
             if stream["choices"][0]["finish_reason"] is None:
                 yield stream["choices"][0]["completion_text"], stream["choices"][0]["completion_tokens"]
             else:
-                logger.info(f'finish_reason {stream["choices"][0]["finish_reason"]} with text: {stream["choices"][0]["text"]}')
+                logger.info(
+                    f'finish_reason {stream["choices"][0]["finish_reason"]} with text: {stream["choices"][0]["text"]}')
 
         # warmup for next turn (下轮解码的加速)
         if suffix_tokens:
@@ -204,11 +195,28 @@ class Qwen2Simulator(Simulator):
             self.llm.eval([151645, 198] + suffix_tokens)  # 增加 n_tokens
             logger.info(f"after warmup: n_tokens = {self.llm.n_tokens}")
 
+    def pre_cache_system(self, system_list):
+        """
+        :param system_list:
+        :return:
+        """
+        logger.info(f"cache size {self.llm.cache.cache_size}")
+        for system_prompt in system_list:
+            logger.info(f"pre caching {system_prompt}")
+            input_ids = self.tokenize(f"<|im_start|>system{system_prompt}<|im_end|>\n<|im_start|>user\n")
+            output = self.llm.create_completion(
+                input_ids,
+                stream=False,
+                max_tokens=3,
+                top_k=1
+            )
+            logger.info(f"cache size {self.llm.cache.cache_size}")
 
+        # disable cache after
+        llama_cpp.LlamaRAMCache.__setitem__ = lambda *args: None
 
-
-
-
+    def complete(self):
+        pass
 
 
 bot = Qwen2Simulator()