Tonic commited on
Commit
17d3814
1 Parent(s): 714fc08

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +89 -0
app.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces
2
+ import gradio as gr
3
+ import torch
4
+ from transformers import AutoModelForCausalLM, AutoTokenizer
5
+ import os
6
+
7
+ title = """# Welcome to 🌟Tonic's🐇🥷🏻Trinity
8
+ You can build with this endpoint using🐇🥷🏻Trinity available here : [WhiteRabbitNeo/Trinity-13B](https://huggingface.co//WhiteRabbitNeo/Trinity-13B). You can also use 🐇🥷🏻Trinity by cloning this space. Simply click here: <a style="display:inline-block" href="https://huggingface.co/spaces/Tonic/trinity?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a></h3>
9
+ Join us : 🌟TeamTonic🌟 is always making cool demos! Join our active builder's 🛠️community 👻 [![Join us on Discord](https://img.shields.io/discord/1109943800132010065?label=Discord&logo=discord&style=flat-square)](https://discord.gg/GWpVpekp) On 🤗Huggingface:[MultiTransformer](https://huggingface.co/MultiTransformer) Math 🔍 [introspector](https://huggingface.co/introspector) On 🌐Github: [Tonic-AI](https://github.com/tonic-ai) & contribute to🌟 [SciTonic](https://github.com/Tonic-AI/scitonic)🤗Big thanks to Yuvi Sharma and all the folks at huggingface for the community grant 🤗
10
+ """
11
+
12
+
13
+ default_system_prompt = """
14
+ Answer the Question by exploring multiple reasoning paths as follows:
15
+ - First, carefully analyze the question to extract the key information components and break it down into logical sub-questions. This helps set up the framework for reasoning. The goal is to construct an internal search tree.
16
+ - For each sub-question, leverage your knowledge to generate 2-3 intermediate thoughts that represent steps towards an answer. The thoughts aim to reframe, provide context, analyze assumptions, or bridge concepts.
17
+ - Evaluate the clarity, relevance, logical flow and coverage of concepts for each thought option. Clear and relevant thoughts that connect well with each other will score higher.
18
+ - Based on the thought evaluations, deliberate to construct a chain of reasoning that stitches together the strongest thoughts in a natural order.
19
+ - If the current chain is determined to not fully answer the question, backtrack and explore alternative paths by substituting different high-scoring thoughts.
20
+ - Throughout the reasoning process, aim to provide explanatory details on thought process rather than just state conclusions, including briefly noting why some thoughts were deemed less ideal.
21
+ - Once a reasoning chain is constructed that thoroughly answers all sub-questions in a clear, logical manner, synthesize the key insights into a final concise answer.
22
+ - Please note that while the focus is on the final answer in the response, it should also include intermediate thoughts inline to illustrate the deliberative reasoning process.
23
+ In summary, leverage a Tree of Thoughts approach to actively explore multiple reasoning paths, evaluate thoughts heuristically, and explain the process - with the goal of producing insightful answers.
24
+ """
25
+
26
+ model_path = "/home/migel/models/WhiteRabbitNeo"
27
+
28
+ hf_token = os.getenv("HF_TOKEN")
29
+ if not hf_token:
30
+ raise ValueError("Hugging Face token not found. Please set the HF_TOKEN environment variable.")
31
+
32
+ model = AutoModelForCausalLM.from_pretrained(
33
+ model_path,
34
+ torch_dtype=torch.float16,
35
+ device_map="auto",
36
+ load_in_8bit=True,
37
+ trust_remote_code=True,
38
+ )
39
+
40
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
41
+
42
+ @spaces.GPU
43
+ def generate_text(custom_prompt, user_input, temperature, generate_len, top_p, top_k):
44
+ system_prompt = custom_prompt if custom_prompt else default_system_prompt
45
+ llm_prompt = f"{system_prompt} \nUSER: {user_input} \nASSISTANT: "
46
+
47
+ tokens = tokenizer.encode(llm_prompt, return_tensors="pt")
48
+ tokens = tokens.to("cuda")
49
+
50
+ length = tokens.shape[1]
51
+ with torch.no_grad():
52
+ output = model.generate(
53
+ input_ids=tokens,
54
+ max_length=length + generate_len,
55
+ temperature=temperature,
56
+ top_p=top_p,
57
+ top_k=top_k,
58
+ num_return_sequences=1,
59
+ )
60
+ generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
61
+ answer = generated_text[len(llm_prompt):].strip()
62
+
63
+ return answer
64
+
65
+ def gradio_app():
66
+ with gr.Blocks() as demo:
67
+ gr.Markdown(title)
68
+ with gr.Row():
69
+ custom_prompt = gr.Textbox(label="Custom System Prompt (optional)", placeholder="Leave blank to use the default prompt...")
70
+ instruction = gr.Textbox(label="Your Instruction", placeholder="Type your question here...")
71
+ with gr.Row():
72
+ temperature = gr.Slider(minimum=0.1, maximum=1.0, step=0.1, value=0.5, label="Temperature")
73
+ generate_len = gr.Slider(minimum=100, maximum=1024, step=10, value=100, label="Generate Length")
74
+ top_p = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=1.0, label="Top P")
75
+ top_k = gr.Slider(minimum=0, maximum=100, step=1, value=50, label="Top K")
76
+ with gr.Row():
77
+ generate_btn = gr.Button("Generate")
78
+ output = gr.Textbox(label="Generated Text", lines=10, placeholder="Generated answer will appear here...")
79
+
80
+ generate_btn.click(
81
+ fn=generate_text,
82
+ inputs=[custom_prompt, instruction, temperature, generate_len, top_p, top_k],
83
+ outputs=output
84
+ )
85
+
86
+ demo.launch()
87
+
88
+ if __name__ == "__main__":
89
+ gradio_app()