Updated app.py to use GGUF model

app.py

@@ -1,27 +1,13 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-import torch
+from llama_cpp import Llama
 import gradio as gr
 
-model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+model_path = "matrixportal/Mistral-7B-Instruct-v0.3-Q4_K_M-GGUF"  # Update with the correct filename
 
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    torch_dtype=torch.float16,
-    low_cpu_mem_usage=True
-)
-
-pipe = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
-    device=0
-)
+llm = Llama(model_path=model_path, n_ctx=2048, n_threads=8)
 
 def chat_response(prompt):
-    response = pipe(prompt, max_new_tokens=100, do_sample=True, temperature=0.7)
-    return response[0]["generated_text"]
+    output = llm(prompt, max_tokens=200, temperature=0.7)
+    return output["choices"][0]["text"]
 
 iface = gr.Interface(fn=chat_response, inputs="text", outputs="text")
-iface.launch(server_name="0.0.0.0", server_port=7860)
+iface.launch()