updated code

app.py

@@ -1,85 +1,100 @@
 import os
 import openai
 import gradio as gr
-import time
+import json
 import requests
 import shutil
-import json
+import random
+import time
 
-from PIL import Image
 from gradio_client import Client
 from newsapi import NewsApiClient
 
-# Import langchain things that are needed generically
+from PIL import Image
+import matplotlib.pyplot as plt
+
+# import all defined functions, their definitions and a dictionary
+from gpt_function_definitions import generate_image, generate_caption, get_news, bored_api
+
+#OpenaI Chat Completions endpoint
+API_URL = "https://api.openai.com/v1/chat/completions" #os.getenv("API_URL") + "/generate_stream"
+
+# Import things that are needed generically from langchain
 from langchain import LLMMathChain, SerpAPIWrapper
-from langchain.agents import AgentType, initialize_agent
+from langchain.agents import AgentType, initialize_agent, load_tools
 from langchain.chat_models import ChatOpenAI
 from langchain.tools import BaseTool, StructuredTool, Tool, tool
-
-from langchain.tools import format_tool_to_openai_function
+from langchain.tools import MoveFileTool, format_tool_to_openai_function
 from langchain.schema import (
     AIMessage,
     HumanMessage,
     SystemMessage
 )
-# import all defined functions, their definitions and a dictionary
-from gpt_function_definitions import generate_image, generate_caption, get_news
-
+from langchain.utilities import WikipediaAPIWrapper
+from langchain.tools import AIPluginTool
 
 # Get the value of the openai_api_key from environment variable
+os.environ['OPENAI_API_KEY'] = "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"
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
-
-#Streaming endpoint
-API_URL = "https://api.openai.com/v1/chat/completions" #os.getenv("API_URL") + "/generate_stream"
+os.environ['SERPAPI_API_KEY'] = "e771e9e02fdace5cbc95249ec3dfa8ab32f9e378cf4c9ad080444c7c0e80b8a6"
 search = SerpAPIWrapper()
 
 
-# TOOLS and FUNCTION CALLING
+# LANGCHAIN
+
 # Load the tool configs that are needed.
-# 'Tool' dataclass wraps functions that accept a single string input and returns a string output.
+# Langchain 'Tool' dataclass wraps functions that accept a single string input and returns a string output.
 tools = [
+    #image generation 
     Tool.from_function(
         func=generate_image,
         name="generate_image",
         description="generate an image based on the prompt provided"
         # coroutine= ... <- you can specify an async method if desired as well
     ),
-    #Tool.from_function(
-    #    func=generate_music,
-    #    name="generate_music",
-    #    description="generate music based on an input text and input melody"
-    #    # coroutine= ... <- you can specify an async method if desired as well
-    #),
+
+    # Describe an image
     Tool.from_function(
         func=generate_caption,
         name="generate_caption",
         description="generate caption for the image present at the filepath provided"
         # coroutine= ... <- you can specify an async method if desired as well
     ),
+
+    # Get lattest top news
     Tool.from_function(
         func=get_news,
         name="get_news",
         description="get top three engilsh news items for a given query, sorted by relevancy"
         # coroutine= ... <- you can specify an async method if desired as well
     ),
+
+    # Search the web using Google search 
     Tool.from_function(
         func=search.run,
         name="Search",
         description="useful for when you need to answer questions about current events"
         # coroutine= ... <- you can specify an async method if desired as well
-    ),    
-]
+    ),   
+
+    #The Bored API
+    Tool.from_function(
+    func=bored_api,
+    name="bored_api",
+    description="Get a random activity to do based on the activity type"
+    # coroutine= ... <- you can specify an async method if desired as well
+    ),  
+    ]
 
 
-# function calling
+# Handling Plugin converations
 def run_conversation(user_input, plugins, tools, chat):
     
     print(f"Plugins are - {plugins}")
     print(f"Total available PLUGINS/Tools are - {tools}")
     
     # Load the tool configs that are needed.
-    # 'Tool' dataclass wraps functions that accept a single string input and returns a string output.
     tools = [val for val, flag in zip(tools, plugins) if flag]
     print(f"PLUGINS/Tools enabled in this run are - {tools}")
 
@@ -91,7 +106,7 @@ def run_conversation(user_input, plugins, tools, chat):
         function_response = agent.run(user_input)
         print(f"function_response is - {function_response}")
             
-        image_file_extns = ['png', 'jpg', 'gif', 'tiff', 'tif', 'svg', 'bmp']
+        image_file_extns = ['.png', '.jpg', '.gif', '.tiff', '.tif', '.svg', '.bmp']
         literal_terms = ['caption', 'captions']
         if any(extn in function_response for extn in image_file_extns) and not any(term in function_response for term in literal_terms) :
             image_file = function_response.replace('sandbox:',"").split('(')[-1].split(')')[0]
@@ -105,9 +120,11 @@ def run_conversation(user_input, plugins, tools, chat):
         return None, None
 
 
-system = SystemMessage(content = "You are a helpful AI assistant") 
+# Setting up a system message for our Chatbot
+system = SystemMessage(content = "You are a helpful AI assistant") # that translates English to Pirate English.")
 
-def predict(user_input, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot):
+# driver
+def predict(user_input, temperature, stable_diff, image_cap, top_news, google_search, bored, file_output, chatbot):
 
     print(f"chatbot - {chatbot}")
     print(f"user_input - {user_input}")
@@ -128,7 +145,7 @@ def predict(user_input, temperature, stable_diff, image_cap, top_news, google_se
     model='gpt-3.5-turbo-0613')
     messages = [system]
     # image, caption, news, serach
-    plugins = [stable_diff, image_cap, top_news, google_search] 
+    plugins = [stable_diff, image_cap, top_news, google_search, bored] 
     function_call_decision = True if any(plugins) else False 
 
     if len(chatbot) != 0:
@@ -174,6 +191,7 @@ def predict(user_input, temperature, stable_diff, image_cap, top_news, google_se
     return "", chatbot, None #"", chatbot
 
 
+# Helper functions for file handling
 def add_image(file_to_save, file_output):
     print(f"image file_to_save is - {file_to_save}")
     print(f"files available in directory are -{file_output}")
@@ -186,7 +204,6 @@ def add_image(file_to_save, file_output):
     print(f"Logging: Updated file directory - {file_output}")
     return file_output #gr.update(value="dog1.jpg")
 
-
 def add_audio(file_to_save, file_output):
     print(f"audio file_to_save is - {file_to_save}")
     print(f"files available in directory are -{file_output}")
@@ -199,7 +216,6 @@ def add_audio(file_to_save, file_output):
     print(f"Logging: Updated file directory - {file_output}")
     return file_output #gr.update(value="dog1.jpg")
 
-
 def upload_file(file, file_output):
     print(f"Logging: all files available - {file_output}")
     print(f"Logging: file uploaded is - {file}")
@@ -213,105 +229,166 @@ def upload_file(file, file_output):
     return file_output
 
 
+# What is happening with function calling, langchain, and Gradio
 messaging = """
-How does a Language Model like GPT makes discerning choices regarding which plugins to run? Well, this is done using the Language Model as a reasoning agent and allowing it to assess and process information intelligently.<br>
-<b>Langchain & OpenAI Function Calling</b>: AI models like gpt-3.5-turbo-0613 and gpt-4-0613, are designed to identify when and how to activate functions through API calls. These function-specific APIs generate a JSON object with necessary arguments, aiming to surpass the efficacy of traditional chat or text completion APIs.<br>
-<b>Gradio Chatbots</b>: Gradio provides super easy way to build Chatbot UI. Refer our <a href="https://gradio.app/docs/#chatbot" target="_blank">Docs</a>. Using Langchain's OpenAI Functions Agent you can create chatbots designed to respond to queries by communicating with external APIs. The API responses are fed back to the Language Model for processing and a new response is generated for the user.The versatility of using Gradio to build LLM applications is immense. FOr example, in this Gradio app, you can have an array of Plugins based on functions which are tailored for various purposes (image, video, audio, text generation, utilities etc). This enhancing the breadth and depth of interactions with your Language Model.
+How does a Language Model like GPT makes discerning choices regarding which plugins to run? Well, this is done using the Language Model as a reasoning agent and allowing it to assess and process information intelligently.<br><br>
+- <b>Langchain & OpenAI Function Calling</b>: AI models like gpt-3.5-turbo-0613 and gpt-4-0613, are designed to identify when and how to activate functions through API calls. These function-specific APIs generate a JSON object with necessary arguments, aiming to surpass the efficacy of traditional chat or text completion APIs.<br><br>
+- <b>Gradio Chatbots</b>: Gradio provides super easy way to build Chatbot UI. Refer our <a href="https://gradio.app/docs/#chatbot" target="_blank">Docs</a>. Using Langchain's OpenAI Functions Agent you can create chatbots designed to respond to queries by communicating with external APIs. The API responses are fed back to the Language Model for processing and a new response is generated for the user.The versatility of using Gradio to build LLM applications is immense. FOr example, in this Gradio app, you can have an array of Plugins based on functions which are tailored for various purposes (image, video, audio, text generation, utilities etc). This enhancing the breadth and depth of interactions with your Language Model.
 """
 
-add_plugin_steps = """## Steps to add new Plugins to your Gradio ChatGPT Chatbot
+
+# How to use this Demo effectively
+howto = """
+Welcome to the <b>ChatGPT-Plugins WebUI</b>, built using Gradio and Langchain! This interactive gradio chatbot uses the GPT3.5-turbo-0613 model from OpenAI and boasts the ability to USE, as well as BUILD Custom Plugins to enhance your chatbot experience.
+<br>Here’s a quick guide for you to get you started:<br><br>
+<b>To get Started</b>: Simply type your messages in the textbox to chat with ChatGPT and press enter!<br><br>
+<b>How to use Plugins</b>: Plugins are provided as checkboxes. If you want to try out a plugin just select that checkbox<br><br>
+
+- <b>DIFFUSERS PLUGIN:</b><br>
+<b>What it does:</b> Generates images based on your text prompt.<br>
+<b>How to use:</b> Type a prompt for the image you want to generate, and the Diffusers plugin will create it for you.<br>
+<b>Example input:</b> "Generate an image of a sunset over the mountains."<br><br>
+
+- <b>IMAGE CAPTION PLUGIN:</b><br>
+<b>What it does:</b> Describes images that you upload.<br>
+<b>How to use:</b> Upload an image using the 'Upload' button. Ask ChatGPT to describe the image make sure to mention the image name to it.<br>
+<b>Example input:</b> "Describe the image cat2.jpg."<br><br>
+
+- <b>NEWS PLUGIN:</b><br>
+<b>What it does:</b> Provides the top 3 news articles based on your search query.<br>
+<b>How to use:</b> Just type in a search query and the NewsAPI plugin will present the top 3 news based on relevance.<br>
+<b>Example input:</b> "Show me the top news about space exploration."<br><br>
+
+- <b>SEARCH PLUGIN:</b><br>
+<b>What it does:</b> Searches internet for your queries. Now you don;t need to limit yourself to a knowledge cut-off of 2021<br>
+<b>How to use:</b> Type in a user message in the chatbot. Google Search plugin will search the internet and present a concise resuklt for you like magic!<br>
+<b>Example input:</b> "Who is the current girlfriend of Leonardo Di Caprio."<br><br>
+
+- <b>BORED API PLUGIN:</b><br>
+<b>What it does:</b> Suggests you activities of different types.<br>
+<b>How to use:</b> Mention that you are bored and want some activities to do or simply ask to generate an activity.<br>
+<b>Example input:</b> "Can you suggest me something to do, I am totally bored."<br><br>
+
+Access Generated Content: Find all generated images in the Gradio Files component located below the input textbox.<br><br>
+Have Fun!: Explore and enjoy the versatile features of this <b>ChatGPT-Plugin WebUI</b>.<br>
+Now you’re all set to make the most of this ChatGPT demo. Happy chatting!
+"""
+
+
+# Guide to add new Plugins
+add_plugin_steps = """
+## Steps to add new Plugins to your Langchain-Gradio ChatGPT PLUGIN WebUI
 
 1. **Acquire the API Endpoint**
-    - You need an API which you can query, and for this example let's consider using a text-to-speech demo hosted on Huggingface Spaces.
-    - **API Endpoint**: [https://gradio-neon-tts-plugin-coqui.hf.space/](https://gradio-neon-tts-plugin-coqui.hf.space/)
+    - You need an API which you can query, and for this example let's consider using a The Bored API.
+    - **API Endpoint**: [https://www.boredapi.com/api/activity/?type=](https://www.boredapi.com/api/activity/?type=)
 
 2. **Create a Function to Query the API**
     - You can access any Gradio demo as an API via the Gradio Python Client.
     ```python
-    from gradio.client import Client
-
-    def texttospeech(input_text):
-        client = Client("https://gradio-neon-tts-plugin-coqui.hf.space/")
-        result = client.predict(
-            input_text,	# str  in 'Input' Textbox component
-            "en",	    # str  in 'Language' Radio component
-            api_name="/predict"
-        )
-        return result
+    def bored_api(activity_type) -> str:
+      '''
+      Get a random activity to do based on the activity type. 
+      '''
+      activity_type_list = ["education", "recreational", "social", "diy", "charity", "cooking", "relaxation", "music", "busywork"]
+      activity_type = activity_type.lower()
+      if activity_type not in activity_type_list:
+        activity_type = random.choice(activity_type_list)
+
+      api_url = "https://www.boredapi.com/api/activity/?type=" + activity_type 
+      response = requests.get(
+          api_url 
+          )
+      return response.json()['activity']
     ```
 
-3. **Describe the Function to GPT-3.5**
-    - You need to describe your function to GPT3.5/4. This function definition will get passed to gpt and will suck up your token. GPT may or may not use this function based on user inputs later on.
-    - You can either use the Gradio demo for converting any given function to the required JSON format for GPT-3.5.
-      - Demo: [Function to JSON](https://huggingface.co/spaces/ysharma/function-to-JSON)
-      - Or, you can create the dictionary object on your own. Note that, the correct format is super important here.
-    - MAke sure to name your JSON object description as `<function_name>_func`.
+3. **Add Function definitions**
+    - Add the function definition to the `gpt_function_definitions.py` file (simply copy and paste). Don't forget to add function description in docstring. 
+    - Add required imports
     ```python
-    texttospeech_func = {
-        "name": "texttospeech",
-        "description": "generate speech from the given input text",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "input_text": {
-                    "type": "string",
-                    "description": "text that will be used to generate speech"
-                }
-            },
-            "required": [
-                "input_text"
-            ]
-        }
-    }
+    from gpt_function_definitions import generate_image, generate_caption, get_news, bored_api
+
     ```
+    
+4. **Add the function to the Tools list**
+    - Add a description - describe what your function does. Models like GPT3.5/4 support Function Calling. The OpenAI Functions Agent from Langchain is designed to work with these functions and models. 
+    - Name - add a name of your function, don't include spaces
 
-4. **Add Function and JSON Object Details**
-    - Add the function definition and description to the `gpt_function_definitions.py` file (simply copy and paste).
-    - `dict_plugin_functions` is a dictionary of all available plugins. Add your plugin information to this dictionary in the required format.
     ```python
-    'texttospeech_func': {
-        'dict': texttospeech_func,
-        'func': texttospeech
-    }
+    tools = [
+        #image generation 
+        ...
+
+        # Describe an image
+        ...
+
+        # Get lattest top news
+        ...
+
+        # Bored Api 
+        Tool.from_function(
+        func=bored_api,
+        name="bored_api",
+        description="Get a random activity to do based on the activity type"
+        # coroutine= ... <- you can specify an async method if desired as well
+        ),  
+    ]
     ```
 
 5. **Update the Chatbot Layout**
     - Go to the Blocks Chatbot layout and add a new checkbox for your plugin as:
     ```python
-    texttospeech = gr.Checkbox(label="📝🗣️Text-To-Speech", value=False)
+    bored = gr.Checkbox(label="🙄bored", value=False)
     ```
-    - Add the new checkbox component to your submit and click events for your chatbot and to the predict function accordingly.
+    - Add the new checkbox component (example - <i>bored</i>) to your submit and click events for your chatbot and to the predict function accordingly.
     - And also to the `plugins` list in `predict`
       ```python
-      plugins = [music_gen, stable_diff, image_cap, top_news, texttospeech]
+      plugins = [stable_diff, image_cap, top_news, search, bored]
       ```
 
-Thats it! you are have added your own brand new CHATGPT Plugin for yourself. Go PLAY!!
+**Thats it! you have added your own brand new CHATGPT Plugin for yourself. Go PLAY!!**
 """
 
 
-# GRADIO BLOCK
+second_headline = """<h3 align="center">🔥This Plugins WebUI is build using <a href="https://www.gradio.app/" target="_blank">Gradio</a>, 
+<a href="https://python.langchain.com/docs/get_started/introduction.html" target="_blank">Langchain</a>, 
+and ChatGPT <a href="https://openai.com/blog/function-calling-and-other-api-updates" target="_blank">Function Calling API</a>. 
+You don't need an OPENAI API key to run this demo as Huggingface is provided one for the community use🙌</h1>"""
+
+
+# Gradio block
 with gr.Blocks(css = """#col_container { margin-left: auto; margin-right: auto;}
-                #chatbot {height: 520px; overflow: auto;}""") as demo:   # #width: 1000px;
-    gr.HTML('<h1 align="center">🚀ChatGPT🧩Plugin WebUI using Langchain & Gradio</h1>')
+                #chatbot {height: 520px; overflow: auto;}""") as demo:   
+                
+    gr.HTML('<h1 align="center">🚀ChatGPT-Plugins🧩 WebUI using Langchain & Gradio</h1>')
+    gr.HTML(second_headline)
+    gr.HTML('''<center><a href="https://huggingface.co/spaces/ysharma/ChatGPT-Plugins-UI-with-Langchain?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>Duplicate the Space and run securely with your OpenAI API Key</center>''')
+    
+    with gr.Accordion("Follow these Steps to use the Gradio WebUI OR simply Click any of the given Examples! ", open=False):
+      gr.HTML(howto)
     with gr.Accordion("What is happening?", open=False):
       gr.HTML(messaging)
-    gr.HTML('''<center><a href="https://huggingface.co/spaces/ysharma/ChatGPT-Plugins-UI-with-Langchain?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>Duplicate the Space and run securely with your OpenAI API Key</center>''')
+    
+    gr.HTML("""Bonus! Steps to build and add your own ChatGPT Plugins to the WebUI using Langchain : <a href="https://huggingface.co/spaces/ysharma/ChatGPT-Plugins-UI-with-Langchain/blob/main/README.md" target="_blank">Add new Plugins to ChatGPT WebUI in 5 mins!!</a>""")
+
     with gr.Row():
       with gr.Column():
         openai_api_key_tb = gr.Textbox(label="Enter your OpenAI API key here",
                                          value="🎁ChatGPT Keys are provided by HuggingFace for Free🥳 You don't need to enter yours!😉🙌",
                                          container=False)
-          #plugin_message = gr.HTML()
+        #plugin_message = gr.HTML()
+        
         with gr.Accordion("Plugins🛠️ Available",open=True):
           with gr.Row():
-            #music_gen = gr.Checkbox(label="🎵MusicGen", value=False)
             stable_diff = gr.Checkbox(label="🖼️Diffusers", value=False)
             image_cap = gr.Checkbox(label="🎨Describe Image", value=False)
             top_news = gr.Checkbox(label="📰News", value=False)
             google_search = gr.Checkbox(label="🌐Google Search", value=False)
+            bored = gr.Checkbox(label="🙄Bored API", value=False)
+            #music_gen = gr.Checkbox(label="🎵MusicGen", value=False)
             #texttospeech = gr.Checkbox(label="📝🗣️Text-To-Speech", value=False)
             #gr.CheckboxGroup(["🎵MusicGen", "🖼️Diffusers", "🎨Describe Image", "📰News", "📝🗣️Text-To-Speech" ], label="Plug-ins", info="enhance your ChatGPT experience using Plugins : Powered by Gradio!")
+
       with gr.Column():
         gen_image = gr.Image(label="generated image", type="filepath", interactive=False)
 
@@ -334,30 +411,32 @@ with gr.Blocks(css = """#col_container { margin-left: auto; margin-right: auto;}
         file_output = gr.File(file_count="multiple", file_types=["image", "audio"], label="Files Available")
 
     inputs.submit( predict,
-                  [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot],
+                  [inputs, temperature, stable_diff, image_cap, top_news, google_search, bored, file_output, chatbot],
                    [inputs, chatbot, gen_image ])
     b1.click( predict,
-            [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot],
+            [inputs, temperature, stable_diff, image_cap, top_news, google_search, bored, file_output, chatbot],
              [inputs, chatbot, gen_image ])
 
 
     btn.upload(upload_file, [btn, file_output], file_output)
     gen_image.change(add_image, [gen_image, file_output], file_output)
     #gen_audio.change(add_audio, [gen_audio, file_output], file_output)
-    gr.HTML("""<a href="https://huggingface.co/spaces/ysharma/ChatGPT-Plugins-in-Gradio/blob/main/README.md" target="_blank">How to add new ChatGPT Plugins in Gradio Chatbot in 5 mins!! or open the accordion below.</a>""")
-    with gr.Accordion("How to add more Plugins to ChatGPT", open=False ):
-      gr.Markdown(add_plugin_steps)
 
-    gr.Examples(
-        examples = [["generate an image of a puppy", 1.0, True, False, False, False, None],
-                    ["generate a caption for the image cat2.jpg", 1.0, False, True, False, False, "cat2.jpg"],
-                    ["What is the latest top news on Inflation in Europe", 1.0, False, False, True, False, None],
-                    ["What is Europe's stand on the ongoing generative AI revolution?", 1.0, False, False, False, True, None],
-                    ["Write a very short poem on 'sparkling water'", 1.0, False, False, False, False, None],
-                    ["What is the weather in LA and SF?", 1.0, False, False, False, True, None],
-                    ["Who is the owner of Twitter? Are there any competitors of Twitter yet?", 1.0, True, True, True, True, None],
+    gr.HTML("<br><br>")
+    gr.Examples(label = "To get started quickly - Click on any example below and press Enter/Run:",
+        examples = [["What is the latest top news on Inflation in Europe", 1.0, False, False, True, False, False, None],
+                    ["What is Europe's stand on the ongoing generative AI revolution?", 1.0, False, False, False, True, False, None],
+                    ["Write a very short poem on 'sparkling water'", 1.0, False, False, False, False, False, None],
+                    ["What is the weather in LA and SF?", 1.0, False, False, False, True, False, None],
+                    ["generate an image of a puppy", 1.0, True, False, False, False, False,None],
+                    ["generate a caption for the image cat2.jpg", 1.0, False, True, False, False, False, "cat2.jpg"],
+                    ["Who is the owner of Twitter? Are there any competitor of Twitter yet?", 1.0, True, True, True, True, False, None],
+                    ["Can you suggest me something to do, I am totally bored", 1.0, False, False, False, False, True, None]
                     ],
-        inputs = [inputs, temperature, stable_diff, image_cap, top_news, google_search, file_output, chatbot]
+        inputs = [inputs, temperature, stable_diff, image_cap, top_news, google_search, bored, file_output]
     )
+    
+    with gr.Accordion("Use Langchain to build and add your own Plugins to this ChatGPT WebUI", open=False ):
+      gr.Markdown(add_plugin_steps)
 
-demo.queue().launch(debug=True, height = '1000')
+demo.queue().launch(debug=True) # height = '1000'