Update app.py

app.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
 model_name = "Azurro/APT-1B-Base"
@@ -10,6 +11,7 @@ generator = pipeline(
     "text-generation",
     model=model,
     tokenizer=tokenizer,
+    torch_dtype=torch.bfloat16,
     device_map="auto",
 )
 
@@ -24,7 +26,7 @@ def generate_text(prompt, max_length, temperature, top_k, top_p, beams):
     return output[0]['generated_text']
 
 input_text = gr.inputs.Textbox(label="Input Text")
-max_length = gr.inputs.Slider(1, 200, step=1, default=100, label="Max Length")
+max_length = gr.inputs.Slider(1, 100, step=1, default=30, label="Max Length")
 temperature = gr.inputs.Slider(0.1, 1.0, step=0.1, default=0.8, label="Temperature")
 top_k = gr.inputs.Slider(1, 200, step=1, default=10, label="Top K")
 top_p = gr.inputs.Slider(0.1, 2.0, step=0.1, default=0.95, label="Top P")
@@ -32,4 +34,6 @@ beams = gr.inputs.Slider(1, 20, step=1, default=1, label="Beams")
 
 outputs = gr.outputs.Textbox(label="Generated Text")
 
-gr.Interface(generate_text, inputs=[input_text, max_length, temperature, top_k, top_p, beams], outputs=outputs).launch()
+iface = gr.Interface(generate_text, inputs=[input_text, max_length, temperature, top_k, top_p, beams], outputs=outputs)
+iface.queue(concurrency_count=1)
+iface.launch(max_threads=100)