Spaces:
Runtime error
Runtime error
File size: 1,436 Bytes
b9e24ef ff11a2a 302a52d ff11a2a 7a17c0a 8fdc643 a330329 ff11a2a 7a17c0a ff11a2a 781821b ff11a2a 781821b 0a15ada ff11a2a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
import os
import gradio as gr
import ctransformers
configObj = ctransformers.Config(stop=["\n", 'User'], context_length=2048)
config = ctransformers.AutoConfig(config=configObj, model_type='llama')
config.config.stop = ["\n"]
# path_to_llm = os.path.abspath("llama-2-7b-chat.ggmlv3.q4_1.bin")
llm = ctransformers.AutoModelForCausalLM.from_pretrained("TheBloke/Llama-2-7b-Chat-GGUF", model_file="llama-2-7b-chat.Q4_K_M.gguf", config=config)
def complete(prompt, stop=["User", "Assistant"]):
tokens = llm.tokenize(prompt)
output = ''
for token in llm.generate(tokens):
result = llm.detokenize(token)
output += result
for word in stop:
if word in output:
print('\n')
return output
print(result, end='',flush=True)
print('\n')
return output
title = "llama2-7b-chat-ggml"
description = "This space is an attempt to run the GGUF 4 bit quantized version of 'llama2-7b-chat' on a CPU"
example_1 = "Write a 7 line poem on AI"
example_2 = "Tell me a joke"
examples = [example_1, example_2]
def generate_response(user_input):
prompt = f'User: {user_input}\nAssistant: '
response = complete(prompt)
return response
UI = gr.Interface(
fn=generate_response,
inputs=gr.Textbox(label="User Query", placeholder="Ask your queries here...."),
outputs=gr.Textbox(label="Assistant Response"),
title=title,
description=description,
examples=examples
)
UI.launch() |