update

app.py

@@ -80,7 +80,7 @@ def contains_chinese(string):
 def clear_history(request: gr.Request):
     state = default_conversation.copy()
 
-    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+    return (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
 
 def http_bot(state, topk, max_new_tokens, random_seed, request: gr.Request):
     prompt = after_process_image(state.get_prompt())
@@ -88,10 +88,10 @@ def http_bot(state, topk, max_new_tokens, random_seed, request: gr.Request):
     state.messages[-1][-1] = "▌"
     yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
 
-    if contains_chinese(prompt):
-        state.messages[-1][-1] = "**CURRENTLY WE ONLY SUPPORT ENGLISH. PLEASE REFRESH THIS PAGE TO RESTART.**"
-        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-        return
+    # if contains_chinese(prompt):
+    #     state.messages[-1][-1] = "**CURRENTLY WE ONLY SUPPORT ENGLISH. PLEASE REFRESH THIS PAGE TO RESTART.**"
+    #     yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+    #     return
 
     try:
         data = get_inputs(prompt, images, topk, max_new_tokens, random_seed)
@@ -111,11 +111,105 @@ def http_bot(state, topk, max_new_tokens, random_seed, request: gr.Request):
     state.messages[-1][-1] = state.messages[-1][-1][:-1]
     yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
 
+
+def add_text_http_bot(state, text, image, video, topk, max_new_tokens, random_seed, request: gr.Request):
+    if len(text) <= 0 and (image is None or video is None):
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "", None, None) + (no_change_btn,) * 5
+
+    if image is not None:
+        multimodal_msg = None
+        if '<image>' not in text:
+            text = text + '\n<image>'
+
+        if multimodal_msg is not None:
+            return (state, state.to_gradio_chatbot(), multimodal_msg, None, None) + (
+                no_change_btn,) * 5
+        text = (text, image)
+    
+    if video is not None:
+        num_frames = 4
+        if '<image>' not in text:
+            text = text + '\n<image>' * num_frames
+        text = (text, video)
+
+    state.append_message(state.roles[0], text)
+    state.append_message(state.roles[1], None)
+    state.skip_next = False
+
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+    
+    prompt = after_process_image(state.get_prompt())
+    images = state.get_images()
+    state.messages[-1][-1] = "▌"
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+
+    # if contains_chinese(prompt):
+    #     state.messages[-1][-1] = "**CURRENTLY WE ONLY SUPPORT ENGLISH. PLEASE REFRESH THIS PAGE TO RESTART OR CLEAR HISTORY.**"
+    #     yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+    #     return
+
+    try:
+        data = get_inputs(prompt, images, topk, max_new_tokens, random_seed)
+        output = model.prediction(data, log_dir)
+        print(output)
+        # output = output.replace("```", "")
+
+        state.messages[-1][-1] = output + "▌"
+        yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+        time.sleep(0.03)
+
+    except requests.exceptions.RequestException as e:
+        state.messages[-1][-1] = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+        yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+        return
+
+    state.messages[-1][-1] = state.messages[-1][-1][:-1]
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (enable_btn,) * 5
+
+
+def regenerate_http_bot(state, topk, max_new_tokens, random_seed, request: gr.Request):
+    state.messages[-1][-1] = None
+    state.skip_next = False
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+    
+    prompt = after_process_image(state.get_prompt())
+    images = state.get_images()
+    state.messages[-1][-1] = " "
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+
+    # if contains_chinese(prompt):
+    #     state.messages[-1][-1] = "**CURRENTLY WE ONLY SUPPORT ENGLISH. PLEASE REFRESH THIS PAGE TO RESTART OR CLEAR HISTORY.**"
+    #     yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+    #     return
+
+    try:
+        data = get_inputs(prompt, images, topk, max_new_tokens, random_seed)
+        output = model.prediction(data, log_dir)
+        print(">>>> output:", output)
+        # output = output.replace("```", "")
+
+        state.messages[-1][-1] = output + " "
+        yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5
+        time.sleep(0.03)
+
+    except requests.exceptions.RequestException as e:
+        print(e)
+        state.messages[-1][-1] = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+        yield (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+        return
+
+    state.messages[-1][-1] = state.messages[-1][-1][:-1]
+    yield (state, state.to_gradio_chatbot(), "", None, None) + (enable_btn,) * 5
+
 title_markdown = ("""
 # mPLUG-Owl🦉 (GitHub Repo: https://github.com/X-PLUG/mPLUG-Owl)
+If you like our project, please give us a star on Github for latest update.
 """)
 
 tos_markdown = ("""
+**Notice:** The output is generated by top-k sampling scheme and may involve some randomness. For multiple image and video, we cannot ensure it's performance since only image-text pairs are used during training.
+
 ### Terms of use
 By using this service, users are required to agree to the following terms:
 The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
@@ -135,11 +229,9 @@ version 1.0
 """
 
 def build_demo():
-    #with gr.Blocks(title="mPLUG-Owl🦉", theme=gr.themes.Base(), css=css) as demo:
-    with gr.Blocks(title="mPLUG-Owl🦉") as demo:
+    # with gr.Blocks(title="mPLUG-Owl🦉", theme=gr.themes.Base(), css=css) as demo:
+    with gr.Blocks(title="mPLUG-Owl🦉", css=css) as demo:
         state = gr.State()
-        with gr.Box():
-            gr.Markdown(SHARED_UI_WARNING)
 
         gr.Markdown(title_markdown)
 
@@ -148,6 +240,8 @@ def build_demo():
 
                 imagebox = gr.Image(type="pil")
 
+                videobox = gr.Video()
+
                 with gr.Accordion("Parameters", open=True, visible=False) as parameter_row:
                     topk = gr.Slider(minimum=1, maximum=5, value=5, step=1, interactive=True, label="Top K",)
                     max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
@@ -155,7 +249,7 @@ def build_demo():
                 gr.Markdown(tos_markdown)
 
             with gr.Column(scale=6):
-                chatbot = grChatbot(elem_id="chatbot", visible=False).style(height=550)
+                chatbot = grChatbot(elem_id="chatbot", visible=False).style(height=800)
                 with gr.Row():
                     with gr.Column(scale=8):
                         textbox = gr.Textbox(show_label=False,
@@ -178,9 +272,14 @@ def build_demo():
             [f'examples/laundry.jpeg', 'Why this happens and how to fix it?'],
             [f'examples/ca.jpeg', "What do you think about the person's behavior?"],
             [f'examples/monalisa-fun.jpg', 'Do you know who drew this painting?​'],
-            [f"examples/Yao_Ming.jpeg", "What is the name of the man on the right?"],
+            # [f"examples/Yao_Ming.jpeg", "What is the name of the man on the right?"],
         ], inputs=[imagebox, textbox])
 
+        gr.Examples(examples=[
+            [f"examples/surf.mp4", "What is the man doing?"],
+            [f"examples/yoga.mp4", "What did the woman doing?"],
+        ], inputs=[videobox, textbox])
+
         gr.Markdown(learn_more_markdown)
         url_params = gr.JSON(visible=False)
 
@@ -191,18 +290,20 @@ def build_demo():
             [state], [textbox, upvote_btn, downvote_btn, flag_btn])
         flag_btn.click(flag_last_response,
             [state], [textbox, upvote_btn, downvote_btn, flag_btn])
-        regenerate_btn.click(regenerate, state,
-            [state, chatbot, textbox, imagebox] + btn_list).then(
-            http_bot, [state, topk, max_output_tokens, temperature],
-            [state, chatbot] + btn_list)
-        clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox] + btn_list)
-
-        textbox.submit(add_text, [state, textbox, imagebox], [state, chatbot, textbox, imagebox] + btn_list
-            ).then(http_bot, [state, topk, max_output_tokens, temperature],
-                   [state, chatbot] + btn_list)
-        submit_btn.click(add_text, [state, textbox, imagebox], [state, chatbot, textbox, imagebox] + btn_list
-            ).then(http_bot, [state, topk, max_output_tokens, temperature],
-                   [state, chatbot] + btn_list)
+        # regenerate_btn.click(regenerate, state,
+        #     [state, chatbot, textbox, imagebox] + btn_list).then(
+        #     http_bot, [state, topk, max_output_tokens, temperature],
+        #     [state, chatbot] + btn_list)
+        regenerate_btn.click(regenerate_http_bot, [state, topk, max_output_tokens, temperature],
+            [state, chatbot, textbox, imagebox, videobox] + btn_list)
+        clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox, videobox] + btn_list)
+
+        # textbox.submit(add_text, [state, textbox, imagebox], [state, chatbot, textbox, imagebox] + btn_list
+        #     ).then(http_bot, [state, topk, max_output_tokens, temperature],
+        #            [state, chatbot] + btn_list)
+        textbox.submit(add_text_http_bot, [state, textbox, imagebox, videobox, topk, max_output_tokens, temperature], [state, chatbot, textbox, imagebox, videobox] + btn_list)
+
+        submit_btn.click(add_text_http_bot, [state, textbox, imagebox, videobox, topk, max_output_tokens, temperature], [state, chatbot, textbox, imagebox, videobox] + btn_list)
 
         demo.load(load_demo, [url_params], [state,
             chatbot, textbox, submit_btn, button_row, parameter_row],
@@ -218,7 +319,7 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="0.0.0.0")
     parser.add_argument("--debug", action="store_true", help="using debug mode")
     parser.add_argument("--port", type=int)
-    parser.add_argument("--concurrency-count", type=int, default=100)
+    parser.add_argument("--concurrency-count", type=int, default=4)
     args = parser.parse_args()
     demo = build_demo()
     demo.queue(concurrency_count=args.concurrency_count, status_update_rate=10, api_open=False).launch(server_name=args.host, debug=args.debug, server_port=args.port, share=False)