app.py

import gradio as gr
import time
import json
from cerebras.cloud.sdk import Cerebras
from typing import List, Dict, Tuple, Any, Generator
from tenacity import retry, stop_after_attempt, wait_fixed

def make_api_call(api_key: str, messages: List[Dict[str, str]], max_tokens: int, is_final_answer: bool = False) -> Dict[str, Any]:
    """
    Make an API call to the Cerebras chat completions endpoint with retry logic.
    """
    client = Cerebras(api_key=api_key)
    
    try:
        start_time = time.time()
        response = client.chat.completions.create(
            model="llama3.1-70b",
            messages=messages,
            max_tokens=max_tokens,
            temperature=0.2,
            response_format={"type": "json_object"}
        )
        end_time = time.time()
        
        content = json.loads(response.choices[0].message.content)
        
        # Access time_info attributes directly
        queue_time = response.time_info.queue_time
        prompt_time = response.time_info.prompt_time
        completion_time = response.time_info.completion_time
        total_time = response.time_info.total_time
        
        # Use the provided usage information
        completion_tokens = response.usage.completion_tokens
        
        # Calculate tokens per second using completion tokens
        tokens_per_second = completion_tokens / total_time if total_time > 0 else 0
        
        content['token_info'] = {
            'completion_tokens': completion_tokens,
            'tokens_per_second': tokens_per_second,
            'queue_time': queue_time,
            'prompt_time': prompt_time,
            'completion_time': completion_time,
            'total_time': total_time  # Use total_time as the 'duration'
        }
        
        return content
    except Exception as e:
        if is_final_answer:
            return {"title": "Error", "content": f"Failed to generate final answer. Error: {str(e)}"}
        else:
            return {"title": "Error", "content": f"Failed to generate step. Error: {str(e)}", "next_action": "final_answer"}

def generate_response(api_key: str, prompt: str) -> Generator[Tuple[List[Tuple[str, str]], float, int, float], None, None]:
    """
    Generate a response to the given prompt using a step-by-step reasoning approach.
    This function is now a generator that yields each step as it's generated.
    """
    system_message = """You are an expert AI assistant that explains your reasoning step by step. For each step, provide a title that describes what you're doing in that step, along with the content. Decide if you need another step or if you're ready to give the final answer. Respond in JSON format with 'title', 'content', and 'next_action' (either 'continue' or 'final_answer') keys. USE AS MANY REASONING STEPS AS POSSIBLE. AT LEAST 3. BE AWARE OF YOUR LIMITATIONS AS AN LLM AND WHAT YOU CAN AND CANNOT DO. IN YOUR REASONING, INCLUDE EXPLORATION OF ALTERNATIVE ANSWERS. CONSIDER YOU MAY BE WRONG, AND IF YOU ARE WRONG IN YOUR REASONING, WHERE IT WOULD BE. FULLY TEST ALL OTHER POSSIBILITIES. YOU CAN BE WRONG. WHEN YOU SAY YOU ARE RE-EXAMINING, ACTUALLY RE-EXAMINE, AND USE ANOTHER APPROACH TO DO SO. DO NOT JUST SAY YOU ARE RE-EXAMINING. USE AT LEAST 3 METHODS TO DERIVE THE ANSWER. USE BEST PRACTICES."""
    
    messages = [
        {"role": "system", "content": system_message},
        {"role": "user", "content": prompt},
        {"role": "assistant", "content": "Thank you! I will now think step by step following my instructions, starting at the beginning after decomposing the problem."}
    ]
    
    steps = []
    step_count = 0
    total_thinking_time = 0
    total_completion_tokens = 0
    
    while True:
        step_data = make_api_call(api_key, messages, 300)
        token_info = step_data.pop('token_info', {'completion_tokens': 0, 'tokens_per_second': 0, 'duration': step_data.get('total_time', 0)})
        
        # Use total_time from token_info as the duration
        total_thinking_time += token_info.get('total_time', 0)
        total_completion_tokens += token_info['completion_tokens']
        
        step_count += 1
        step_title = f"Step {step_count}: {step_data['title']}"
        step_content = f"{step_data['content']}\n\n**API Call Duration: {token_info['total_time']:.2f} seconds**\n**Completion Tokens: {token_info['completion_tokens']}, Tokens/s: {token_info['tokens_per_second']:.2f}**"
        steps.append((step_title, step_content))
        messages.append({"role": "assistant", "content": json.dumps(step_data)})
        
        # Calculate the overall average tokens per second using completion tokens
        overall_tokens_per_second = total_completion_tokens / total_thinking_time if total_thinking_time > 0 else 0
        
        # Yield the current conversation, total thinking time, total completion tokens, and overall average tokens per second
        yield steps, total_thinking_time, total_completion_tokens, overall_tokens_per_second
        
        if step_data.get('next_action') == 'final_answer':
            break

    # Request the final answer
    messages.append({"role": "user", "content": "Please provide the final answer based on your reasoning above."})
    
    final_data = make_api_call(api_key, messages, 200, is_final_answer=True)
    token_info = final_data.pop('token_info', {'completion_tokens': 0, 'tokens_per_second': 0, 'duration': final_data.get('total_time', 0)})
    
    total_thinking_time += token_info.get('total_time', 0)
    total_completion_tokens += token_info['completion_tokens']
    
    final_content = f"{final_data.get('content', 'No final answer provided.')}\n\n**Final answer API call duration: {token_info['total_time']:.2f} seconds**\n**Completion Tokens: {token_info['completion_tokens']}, Tokens/s: {token_info['tokens_per_second']:.2f}**"
    steps.append(("Final Answer", final_content))
    
    # Calculate the final overall average tokens per second using completion tokens
    overall_tokens_per_second = total_completion_tokens / total_thinking_time if total_thinking_time > 0 else 0
    
    # Yield the final conversation, total thinking time, total completion tokens, and overall average tokens per second
    yield steps, total_thinking_time, total_completion_tokens, overall_tokens_per_second

def respond(api_key: str, message: str, history: List[Tuple[str, str]]) -> Generator[Tuple[List[Tuple[str, str]], str], None, None]:
    """
    Generator function to handle responses and yield updates for streaming.
    The conversation will now show the newest message at the top.
    """
    if not api_key:
        yield history, "Please provide a valid Cerebras API key."
        return

    # Initialize the generator
    response_generator = generate_response(api_key, message)
    
    for steps, total_time, total_completion_tokens, avg_tokens_per_second in response_generator:
        conversation = history.copy()
        for title, content in steps[len(conversation):]:
            if title.startswith("Step") or title == "Final Answer":
                # Prepend new messages to display newest first
                conversation.insert(0, (title, content))
            else:
                # Prepend any other messages
                conversation.insert(0, (title, content))
        yield conversation, f"**Total API call time:** {total_time:.2f} seconds\n**Completion tokens:** {total_completion_tokens}\n**Overall average tokens/s:** {avg_tokens_per_second:.2f}"

def main():
    with gr.Blocks() as demo:
        gr.Markdown("# o1-like Chain of Thought - LLaMA-3.1 70B on Cerebras")
        gr.Markdown("""
        Implement Chain of Thought with prompting to improve output accuracy.
        Powered by Cerebras, ensuring fast reasoning steps.
        """)
        
        with gr.Row():
            api_key_input = gr.Textbox(
                label="Cerebras API Key", 
                type="password", 
                placeholder="Enter your Cerebras API key", 
                show_label=True
            )
        
        chatbot = gr.Chatbot(label="Conversation")
        with gr.Row():
            user_input = gr.Textbox(
                label="Your Query", 
                placeholder="Enter your query here...", 
                show_label=True
            )
            submit_btn = gr.Button("Submit")
        
        thinking_time_display = gr.Textbox(
            label="Performance Metrics", 
            value="", 
            interactive=False
        )
        
        submit_btn.click(
            fn=respond, 
            inputs=[api_key_input, user_input, chatbot], 
            outputs=[chatbot, thinking_time_display],
            queue=True
        )
        
        # Allow pressing Enter to submit
        user_input.submit(
            fn=respond, 
            inputs=[api_key_input, user_input, chatbot], 
            outputs=[chatbot, thinking_time_display],
            queue=True
        )
    
    demo.launch()

if __name__ == "__main__":
    main()