app.py

import gradio as gr
import os
from math import exp
from tenacity import retry, wait_random_exponential, stop_after_attempt
from langchain_core.messages import HumanMessage, SystemMessage
import tiktoken
from langchain_openai import ChatOpenAI

class OpenAIChat(ChatOpenAI):
    """Openai chatbot LLM."""
    def __init__(
            self,
            model_name: str = "gpt-4o",
            temperature: float = 1,
            top_p: float = 1,
            presence_penalty: float = 0,
            frequency_penalty: float = 0,
            max_tokens: int = 2000,
            cache: bool = False,
            seed: int = 42
    ):
        """Initialize azure chat LLM."""
        super(ChatOpenAI, self).__init__(
            model_name=model_name,
            openai_api_key=os.getenv("openai_api"),
            temperature=temperature,
            
            max_tokens=max_tokens,
            model_kwargs={"presence_penalty": presence_penalty,
                          "frequency_penalty": frequency_penalty,
                          "logprobs": True,
                          "seed": seed,
                          "top_p": top_p,
                          },
            cache=cache
        )



model_name = "gpt-4o"
tokenizer = tiktoken.encoding_for_model("gpt-4o")
tokens = ["Yes", "No"]
ids = [tokenizer.encode(token)[0] for token in tokens]
model = OpenAIChat(model_name=model_name).bind(logit_bias=dict((id, 1) for id in ids))

system_prompt = """
You are an assistant to decide if a statement is as the similar format of the example statements.
In general, a statement has complete information, its positive, 
If the statement is ambigious, often seems like a follow up, or need more backgroud or context to understand, its negative.
If a statement is positive, reply Yes, otherwise reply No. Do not reply the reasoning, just Yes or No.

Here are some positive and negative examples:

[S]: Top ten trucks with best fuel mileage.
[A]: Yes

[S]: Which battery electric vehicle travelled the most distance this year?
[A]: Yes

[S]: Show me a list of all customer visits from yesterday.
[A]: Yes

[S]: What is the average fuel usage per 100km?
[A]: Yes

[S]: Best month with the lowest idle.
[A]: Yes

[S]: Best month with the lowest idle.
[A]: Yes

[S]: What was the idle each month this year?
[A]: Yes


[S]: How about the TwinTaxi?
[A]: No
This one is No since it doesn't specify what thing about the TwinTaxi

[S]: Group services.
[A]: No
This one is No since it doesn't specify thing about Group services.

[S]: Last week.
[A]: No
This one is No since it doesn't specify last week for what?

[S]: Yes, it is.
[A]: No
This one is No since it doesn't specify what thing is confirmed.

[S]: And the mileage?
[A]: No
This one is No since it doesn't specify mileage for what.

[S]: Details, please.
[A]: No
This one is No since it doesn't specify details for what.
"""

prompt = """
Now Answer this statament:
[S]: {statement}
[A]: 
"""


@retry(wait=wait_random_exponential(min=1, max=40), stop=stop_after_attempt(3))
def statement_valid(statement):
    user_prompt = prompt.format(statement=statement)
    messages = [
    SystemMessage(
        content=system_prompt
    ),
    HumanMessage(
        content=user_prompt
    ),
    ]
    response = model.invoke(messages)
    return (
        statement,
        response.content.strip(),
        response.response_metadata['logprobs']['content'][0]['logprob']
    )


def make_pred(statement: str) -> str:
    _, prediction, logprob = statement_valid(statement)
    probability = exp(logprob)
    yes_probability = probability * -1 + 1 if prediction == "No" else probability
    if yes_probability > 0.5:
        return "No need to rephrase, send in the original question."
    return "Need to be rephrased with chat hist"


def get_question_examples():
    examples=[
        "And the mileage?", 
        "I mean last week.", 
        "How about the B123?",
        "Top ten trucks with best fuel mileage.", 
        "What trucks in motion have no driver assigned?", 
        "What group is vehicle Blueberry in?"
    ]
    return examples


with gr.Blocks() as demo:
    with gr.Row(equal_height=False):
        input_statement = gr.Textbox(placeholder="type your statement", label="Prompt")
        reponse = gr.Textbox(placeholder=None, label="AI Response", lines=5)
        gr.Examples(
                    examples=get_question_examples(),
                    inputs=[input_statement],
                    fn=None,
                    outputs=None,
                    cache_examples=False,
                    label="Question Examples"
        ) 
        
    btn = gr.Button("run", variant="primary")

    btn.click(
        fn=make_pred, 
        inputs=[
            input_statement,
        ],
        outputs=[reponse]
    )
demo.launch(share=False)