app.py

import os
import openai
import gradio as gr
import time
import requests
import shutil
import json

from PIL import Image
from gradio_client import Client
from newsapi import NewsApiClient

# Import langchain things that are needed generically
from langchain import LLMMathChain, SerpAPIWrapper
from langchain.agents import AgentType, initialize_agent
from langchain.chat_models import ChatOpenAI
from langchain.tools import BaseTool, StructuredTool, Tool, tool

from langchain.tools import format_tool_to_openai_function
from langchain.schema import (
    AIMessage,
    HumanMessage,
    SystemMessage
)
# import all defined functions, their definitions and a dictionary
from gpt_function_definitions import generate_image, generate_caption, get_news


# Get the value of the openai_api_key from environment variable
openai.api_key = os.getenv("OPENAI_API_KEY")

#chat = ChatOpenAI(
#   #openai_api_key=openai_api_key,
#    temperature=1.0,
#    streaming=True,
#    model='gpt-3.5-turbo-0613'
#    )

#Streaming endpoint
API_URL = "https://api.openai.com/v1/chat/completions" #os.getenv("API_URL") + "/generate_stream"



# TOOLS and FUNCTION CALLING
# Load the tool configs that are needed.
# 'Tool' dataclass wraps functions that accept a single string input and returns a string output.
tools = [
    Tool.from_function(
        func=generate_image,
        name="generate_image",
        description="generate an image based on the prompt provided"
        # coroutine= ... <- you can specify an async method if desired as well
    ),
    #Tool.from_function(
    #    func=generate_music,
    #    name="generate_music",
    #    description="generate music based on an input text and input melody"
    #    # coroutine= ... <- you can specify an async method if desired as well
    #),
    Tool.from_function(
        func=generate_caption,
        name="generate_caption",
        description="generate caption for the image present at the filepath provided"
        # coroutine= ... <- you can specify an async method if desired as well
    ),
    Tool.from_function(
        func=get_news,
        name="get_news",
        description="get top three engilsh news items for a given query, sorted by relevancy"
        # coroutine= ... <- you can specify an async method if desired as well
    ),]

# Creating OpenAI functions
# use LangChain tools as OpenAI functions.
#functions = [format_tool_to_openai_function(t) for t in tools]
#functions

# defining agents using tools and openai functions
#agent = initialize_agent(tools, chat, agent=AgentType.OPENAI_FUNCTIONS, verbose=True)


# function calling
def run_conversation(user_input, plugins, tools, chat):
    
    print(f"Plugins are - {plugins}")
    print(f"Total available PLUGINS/Tools are - {tools}")
    
    # Load the tool configs that are needed.
    # 'Tool' dataclass wraps functions that accept a single string input and returns a string output.
    tools = [val for val, flag in zip(tools, plugins) if flag]
    print(f"PLUGINS/Tools enabled in this run are - {tools}")

    try:
        # defining agents using tools and openai functions
        agent = initialize_agent(tools, chat, agent=AgentType.OPENAI_FUNCTIONS, verbose=True)

        # calling the agent
        function_response = agent.run(user_input)
        print(f"function_response is - {function_response}")
            
        image_file_extns = ['png', 'jpg', 'gif', 'tiff', 'tif', 'svg', 'bmp']
        literal_terms = ['caption', 'captions']
        if any(extn in function_response for extn in image_file_extns) and not any(term in function_response for term in literal_terms) :
            image_file = function_response.replace('sandbox:',"").split('(')[-1].split(')')[0]
            print(f"image_file is -{image_file}")
            return function_response, image_file

        return function_response, None
    
    except Exception as e:
        print(f"An error occured while calling agents using 'Function Calling': {e}")
        return None, None


system = SystemMessage(content = "You are a helpful AI assistant") 

def predict(user_input, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot):

    print(f"chatbot - {chatbot}")
    print(f"user_input - {user_input}")

    # file handling
    print(f"Logging: files in the file directory is -{file_output}")
    if file_output is not None:
      files_avail =  [f.name for f in file_output ]
      print(f"files_available are -{files_avail} ")
    else:
      print("No files available at the moment!")


    chat = ChatOpenAI(
    #openai_api_key=openai_api_key,
    temperature=temperature, #1.0
    streaming=True,
    model='gpt-3.5-turbo-0613')
    messages = [system]
    # image, caption, news, serach
    plugins = [stable_diff, image_cap, top_news, google_search] 
    function_call_decision = True if any(plugins) else False 

    if len(chatbot) != 0:
        for conv in chatbot:
            human = HumanMessage(content=conv[0])
            ai = AIMessage(content=conv[1])
            messages.append(human)
            messages.append(ai)
        messages.append(HumanMessage(content=user_input))
        print(f"messages list is - {messages}")

        if function_call_decision:
            # getting openAI function agent reponse
            function_response, image_file = run_conversation(user_input, plugins, tools, chat)
            if function_response is not None:
                gpt_response = AIMessage(content= function_response)
                bot_message = gpt_response.content
                print(f"bot_message - {bot_message}")
                chatbot.append((user_input, bot_message))
                return "", chatbot, image_file
    else: # for first user message
        messages.append(HumanMessage(content=user_input))
        print(f"messages list is - {messages}")

        if function_call_decision:
            # getting openAI function agent reponse
            function_response, image_file = run_conversation(user_input, plugins, tools, chat)
            if function_response is not None:
                gpt_response = AIMessage(content= function_response)
                bot_message = gpt_response.content
                print(f"bot_message - {bot_message}")
                chatbot.append((user_input, bot_message))
                return "", chatbot, image_file

    # getting gpt3.5's response
    gpt_response = chat(messages)
    print(f"gpt_response - {gpt_response}")
    bot_message = gpt_response.content
    print(f"bot_message - {bot_message}")

    chatbot.append((user_input, bot_message))

    return "", chatbot, None #"", chatbot


def add_image(file_to_save, file_output):
    print(f"image file_to_save is - {file_to_save}")
    print(f"files available in directory are -{file_output}")

    if file_output is not None:
      file_output = [f.name for f in file_output]
    if file_to_save is None:
      return file_output
    file_output = [file_to_save] if file_output is None else file_output + [file_to_save]
    print(f"Logging: Updated file directory - {file_output}")
    return file_output #gr.update(value="dog1.jpg")


def add_audio(file_to_save, file_output):
    print(f"audio file_to_save is - {file_to_save}")
    print(f"files available in directory are -{file_output}")

    if file_output is not None:
      file_output = [f.name for f in file_output]
    if file_to_save is None:
      return file_output
    file_output = [file_to_save] if file_output is None else file_output + [file_to_save]
    print(f"Logging: Updated file directory - {file_output}")
    return file_output #gr.update(value="dog1.jpg")


def upload_file(file, file_output):
    print(f"Logging: all files available - {file_output}")
    print(f"Logging: file uploaded is - {file}")

    img_orig_name = file.name.split('/')[-1]
    shutil.copy2(file.name, img_orig_name)

    file_output = [file] if file_output is None else file_output + [file]
    file_output = [f.name for f in file_output]
    print(f"Logging: Updated file list is - {file_output}")
    return file_output


messaging = """
How does a Language Model like GPT makes discerning choices regarding which plugins to run? Well, this is done using the Language Model as a reasoning agent and allowing it to assess and process information intelligently.<br>
<b>Langchain & OpenAI Function Calling</b>: AI models like gpt-3.5-turbo-0613 and gpt-4-0613, are designed to identify when and how to activate functions through API calls. These function-specific APIs generate a JSON object with necessary arguments, aiming to surpass the efficacy of traditional chat or text completion APIs.<br>
<b>Gradio Chatbots</b>: Gradio provides super easy way to build Chatbot UI. Refer our <a href="https://gradio.app/docs/#chatbot" target="_blank">Docs</a>. Using Langchain's OpenAI Functions Agent you can create chatbots designed to respond to queries by communicating with external APIs. The API responses are fed back to the Language Model for processing and a new response is generated for the user.The versatility of using Gradio to build LLM applications is immense. FOr example, in this Gradio app, you can have an array of Plugins based on functions which are tailored for various purposes (image, video, audio, text generation, utilities etc). This enhancing the breadth and depth of interactions with your Language Model.
"""

add_plugin_steps = """## Steps to add new Plugins to your Gradio ChatGPT Chatbot

1. **Acquire the API Endpoint**
    - You need an API which you can query, and for this example let's consider using a text-to-speech demo hosted on Huggingface Spaces.
    - **API Endpoint**: [https://gradio-neon-tts-plugin-coqui.hf.space/](https://gradio-neon-tts-plugin-coqui.hf.space/)

2. **Create a Function to Query the API**
    - You can access any Gradio demo as an API via the Gradio Python Client.
    ```python
    from gradio.client import Client

    def texttospeech(input_text):
        client = Client("https://gradio-neon-tts-plugin-coqui.hf.space/")
        result = client.predict(
            input_text,	# str  in 'Input' Textbox component
            "en",	    # str  in 'Language' Radio component
            api_name="/predict"
        )
        return result
    ```

3. **Describe the Function to GPT-3.5**
    - You need to describe your function to GPT3.5/4. This function definition will get passed to gpt and will suck up your token. GPT may or may not use this function based on user inputs later on.
    - You can either use the Gradio demo for converting any given function to the required JSON format for GPT-3.5.
      - Demo: [Function to JSON](https://huggingface.co./spaces/ysharma/function-to-JSON)
      - Or, you can create the dictionary object on your own. Note that, the correct format is super important here.
    - MAke sure to name your JSON object description as `<function_name>_func`.
    ```python
    texttospeech_func = {
        "name": "texttospeech",
        "description": "generate speech from the given input text",
        "parameters": {
            "type": "object",
            "properties": {
                "input_text": {
                    "type": "string",
                    "description": "text that will be used to generate speech"
                }
            },
            "required": [
                "input_text"
            ]
        }
    }
    ```

4. **Add Function and JSON Object Details**
    - Add the function definition and description to the `gpt_function_definitions.py` file (simply copy and paste).
    - `dict_plugin_functions` is a dictionary of all available plugins. Add your plugin information to this dictionary in the required format.
    ```python
    'texttospeech_func': {
        'dict': texttospeech_func,
        'func': texttospeech
    }
    ```

5. **Update the Chatbot Layout**
    - Go to the Blocks Chatbot layout and add a new checkbox for your plugin as:
    ```python
    texttospeech = gr.Checkbox(label="📝🗣️Text-To-Speech", value=False)
    ```
    - Add the new checkbox component to your submit and click events for your chatbot and to the predict function accordingly.
    - And also to the `plugins` list in `predict`
      ```python
      plugins = [music_gen, stable_diff, image_cap, top_news, texttospeech]
      ```

Thats it! you are have added your own brand new CHATGPT Plugin for yourself. Go PLAY!!
"""


# GRADIO BLOCK
with gr.Blocks(css = """#col_container { margin-left: auto; margin-right: auto;}
                #chatbot {height: 520px; overflow: auto;}""") as demo:   # #width: 1000px;
    gr.HTML('<h1 align="center">🚀ChatGPT🧩Plugin WebUI using Langchain & Gradio</h1>')
    with gr.Accordion("What is happening?", open=False):
      gr.HTML(messaging)
    gr.HTML('''<center><a href="https://huggingface.co./spaces/ysharma/ChatGPT-Plugins-UI-with-Langchain?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>Duplicate the Space and run securely with your OpenAI API Key</center>''')
    with gr.Row():
      with gr.Column():
        openai_api_key_tb = gr.Textbox(label="Enter your OpenAI API key here",
                                         value="🎁ChatGPT Keys are provided by HuggingFace for Free🥳 You don't need to enter yours!😉🙌",
                                         container=False)
          #plugin_message = gr.HTML()
        with gr.Accordion("Plugins🛠️ Available",open=True):
          with gr.Row():
            #music_gen = gr.Checkbox(label="🎵MusicGen", value=False)
            stable_diff = gr.Checkbox(label="🖼️Diffusers", value=False)
            image_cap = gr.Checkbox(label="🎨Describe Image", value=False)
            top_news = gr.Checkbox(label="📰News", value=False)
            google_search = gr.Checkbox(label="🌐Google Search", value=False)
            #texttospeech = gr.Checkbox(label="📝🗣️Text-To-Speech", value=False)
            #gr.CheckboxGroup(["🎵MusicGen", "🖼️Diffusers", "🎨Describe Image", "📰News", "📝🗣️Text-To-Speech" ], label="Plug-ins", info="enhance your ChatGPT experience using Plugins : Powered by Gradio!")
      with gr.Column():
        gen_image = gr.Image(label="generated image", type="filepath", interactive=False)

    with gr.Row():
        chatbot = gr.Chatbot(elem_id='chatbot')

    with gr.Row():
      with gr.Column(scale=0.70):
        inputs = gr.Textbox(placeholder= "Hi there!", label= "Type an input and press Enter")
      with gr.Column(scale=0.15, min_width=0):
          b1 = gr.Button("🏃Run")
      with gr.Column(scale=0.15, min_width=0):
          btn = gr.UploadButton("📁Upload", file_types=["image", "audio"],  file_count="single")

    with gr.Row():
      with gr.Accordion("Parameters", open=False):
        top_p = gr.Slider( minimum=-0, maximum=1.0, value=1.0, step=0.05, interactive=True, label="Top-p (nucleus sampling)",)
        temperature = gr.Slider( minimum=-0, maximum=5.0, value=1.0, step=0.1, interactive=True, label="Temperature",)
      with gr.Accordion("Available Files", open=False):
        file_output = gr.File(file_count="multiple", file_types=["image", "audio"], label="Files Available")

    inputs.submit( predict,
                  [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot],
                   [inputs, chatbot, gen_image ])
    b1.click( predict,
            [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot],
             [inputs, chatbot, gen_image ])


    btn.upload(upload_file, [btn, file_output], file_output)
    gen_image.change(add_image, [gen_image, file_output], file_output)
    #gen_audio.change(add_audio, [gen_audio, file_output], file_output)
    gr.HTML("""<a href="https://huggingface.co./spaces/ysharma/ChatGPT-Plugins-in-Gradio/blob/main/README.md" target="_blank">How to add new ChatGPT Plugins in Gradio Chatbot in 5 mins!! or open the accordion below.</a>""")
    with gr.Accordion("How to add more Plugins to ChatGPT", open=False ):
      gr.Markdown(add_plugin_steps)

    gr.Examples(
        examples = [["generate an image of a puppy", 1.0, True, False, False, False, None],
                    ["generate a caption for the image cat2.jpg", 1.0, False, True, False, False, "cat2.jpg"],
                    ["What is the latest top news on Inflation in Europe", 1.0, False, False, True, False, None],
                    ["What is Europe's stand on the ongoing generative AI revolution?", 1.0, False, False, False, True, None],
                    ["Write a very short poem on 'sparkling water'", 1.0, False, False, False, False, None],
                    ["What is the weather in LA and SF?", 1.0, False, False, False, True, None],
                    ["Who is the owner of Twitter? Are there any competitors of Twitter yet?", 1.0, True, True, True, True, None],
                    ],
        inputs = [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot]
    )

demo.queue().launch(debug=True, height = '1000')