import gradio as gr import random import time from ctransformers import AutoModelForCausalLM params = { "max_new_tokens":512, "stop":["" ,"<|endoftext|>"], "temperature":0.7, "top_p":0.8, "stream":True, "batch_size": 8} llm = AutoModelForCausalLM.from_pretrained("Lajonbot/WizardLM-13B-V1.2-PL-lora_GGML", model_type="llama") with gr.Blocks() as demo: chatbot = gr.Chatbot() msg = gr.Textbox() clear = gr.Button("Clear") def user(user_message, history): return "", history + [[user_message, None]] def bot(history): stream = llm(prompt = f"JesteÅ› AI assystentem. Odpowiadaj po polski. : {history}. :", **params) history[-1][1] = "" for character in stream: history[-1][1] += character time.sleep(0.005) yield history msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then( bot, chatbot, chatbot ) clear.click(lambda: None, None, chatbot, queue=False) demo.queue() demo.launch()