opentools-->octotools; added remaining tools; polished the ui

octotools/tools/README.md

@@ -0,0 +1,44 @@
+
+## Testing the Tools
+
+To test the text detection tool, follow these steps:
+
+1. **Navigate to the Project Directory:**
+
+   Change your current directory to where the tools are located. Replace `your_path` with the actual path to your project directory.
+
+   ```sh
+   cd your_path/toolbox-agent/octotools
+   ```
+
+2. **Run the Text Detection Tool:**
+
+   ```sh
+   cd toolbox-agent
+   export PYTHONPATH=$(pwd)
+   ```
+
+
+   Execute the tool using the following command:
+
+   ```sh
+   python tools/text_detector/tool.py
+
+   python tools/object_detector/tool.py
+
+   ```
+
+## File Structure
+
+The project is organized as follows:
+
+```sh
+├── __init__.py              # Initializes the tools package and possibly exposes submodules
+├── base.py                  # Base class for tools, providing common functionality
+├── text_detector/           # Directory for the text detection tool
+│   ├── readme.md            # Documentation for the text detection tool
+│   └── tool.py              # Implementation of the text detection tool
+├── object_detector/         # Directory for the object detection tool
+│   ├── readme.md            # Documentation for the object detection tool
+│   └── tool.py              # Implementation of the object detection tool
+```

octotools/tools/advanced_object_detector/tool.py

@@ -0,0 +1,236 @@
+# Grounding DINO Object Detection Tool
+# https://huggingface.co/IDEA-Research/grounding-dino
+
+import os
+import time
+
+from octotools.tools.base import BaseTool
+from PIL import Image, ImageOps
+
+import os
+# Suppress stderr by redirecting it to /dev/null
+import sys
+import re
+import base64
+import requests
+sys.stderr = open(os.devnull, 'w')
+
+
+class Advanced_Object_Detector_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Advanced_Object_Detector_Tool",
+            tool_description="A tool that detects objects in an image using the Grounding DINO-X model and saves individual object images with empty padding.",
+            tool_version="1.0.0",
+            input_types={
+                "image": "str - The path to the image file.",
+                "labels": "list - A list of object labels to detect.",
+                "threshold": "float - The confidence threshold for detection (default: 0.35).",
+                "padding": "int - The number of pixels to add as empty padding around detected objects (default: 20)."
+            },
+            output_type="list - A list of detected objects with their scores, bounding boxes, and saved image paths.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", labels=["baseball", "basket"])',
+                    "description": "Detect baseball and basket in an image, save the detected objects with default empty padding, and return their paths."
+                },
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", labels=["car", "person"], threshold=0.5, model_size="base", padding=15)',
+                    "description": "Detect car and person in an image using the base model, save the detected objects with 15 pixels of empty padding, and return their paths."
+                }
+            ],
+            user_metadata={
+                "limitation": "The model may not always detect objects accurately, and its performance can vary depending on the input image and the associated labels. It typically struggles with detecting small objects, objects that are uncommon, or objects with limited or specific attributes. For improved accuracy or better detection in certain situations, consider using supplementary tools or image processing techniques to provide additional information for verification."
+            }
+        )
+        self.DINO_KEY = os.environ.get("DINO_KEY")
+
+    def preprocess_caption(self, caption):
+        result = caption.lower().strip()
+        if result.endswith("."):
+            return result
+        return result + "."
+
+    def build_tool(self, threshold=0.35):
+
+        params_dict = {
+                        'headers': {
+                                    "Content-Type": "application/json",
+                                    "Token"       : self.DINO_KEY
+                                    },
+                        'body':{
+                                    "image"  : None,
+                                    "prompts": [
+                                        {"type": "text", "text": None},
+                                    ],
+                                    "bbox_threshold": threshold 
+                                }
+
+                      }
+        return params_dict
+
+
+    def save_detected_object(self, image, box, image_name, label, index, padding):
+        object_image = image.crop(box)
+        padded_image = ImageOps.expand(object_image, border=padding, fill='white')
+        
+        filename = f"{image_name}_{label}_{index}.png"
+        os.makedirs(self.output_dir, exist_ok=True)
+        save_path = os.path.join(self.output_dir, filename)
+        
+        padded_image.save(save_path)
+        return save_path
+
+    def execute(self, image, labels, threshold=0.35, padding=20, max_retries=10, retry_delay=5):
+        retry_count = 0
+        params = self.build_tool(threshold)
+
+        def process_image(input_str):
+
+            def image_to_base64(image_path):
+                with open(image_path, "rb") as image_file:
+                    return base64.b64encode(image_file.read()).decode('utf-8')
+            # Define common image file extensions
+            image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.tiff', '.webp'}
+
+            # Check if it is a URL
+            url_pattern = re.compile(r'^(http|https|ftp)://')
+            if url_pattern.match(input_str):
+                if input_str.lower().endswith(tuple(image_extensions)):
+                    return input_str
+                return input_str
+
+            # Check if it is a file path
+            _, ext = os.path.splitext(input_str)
+            if ext.lower() in image_extensions:
+                image_base64 = image_to_base64(input_str)
+                return f'data:image/png;base64,{image_base64}'
+            return None
+
+        if len(labels) < 1:
+            preprocessed_prompt = '<prompt_free>'
+        else:
+            preprocessed_prompt = ''
+            for label in labels:
+                preprocessed_prompt += self.preprocess_caption(label)
+
+
+        body = params['body']
+        body['image'] = process_image(image)
+        body['prompts'] =  [{"type": "text", "text": preprocessed_prompt}]
+
+        # send request
+        resp = requests.post(
+            'https://api.deepdataspace.com/tasks/dinox',
+            json=body,
+            headers=params['headers']
+        )
+
+        if resp.status_code == 200:
+            json_resp = resp.json()
+            print(json_resp)
+
+            # get task_uuid
+            task_uuid = json_resp["data"]["task_uuid"]
+            print(f'task_uuid:{task_uuid}')
+
+            # poll get task result
+            while retry_count < max_retries:
+                resp = requests.get(f'https://api.deepdataspace.com/task_statuses/{task_uuid}', headers=params['headers'])
+                
+
+                if resp.status_code != 200:
+                    break
+                json_resp = resp.json()
+
+                if json_resp["data"]["status"] not in ["waiting", "running"]:
+                    break
+                time.sleep(1)#retry_delay)
+                retry_count += 1
+
+            if json_resp["data"]["status"] == "failed":
+                print(f'failed resp: {json_resp}')
+            elif json_resp["data"]["status"] == "success":
+                # print(f'success resp: {json_resp}')
+                formatted_results = []
+                original_image = Image.open(image)
+                image_name = os.path.splitext(os.path.basename(image))[0]
+                
+                object_counts = {}
+
+                for result in json_resp['data']['result']['objects']:
+                    box = tuple(result["bbox"])
+                    try:
+                        box = [int(x) for x in box]
+                    except:
+                        continue
+                    label = result["category"]
+                    score = round(result["score"], 2)
+                    if label.endswith("."):
+                        label = label[:-1]
+                    
+                    object_counts[label] = object_counts.get(label, 0) + 1
+                    index = object_counts[label]
+
+                    save_path = self.save_detected_object(original_image, box, image_name, label, index, padding)
+            
+                    formatted_results.append({
+                        "label": label,
+                        "confidence score": score,
+                        "box": box,
+                        "saved_image_path": save_path
+                    })
+
+                return formatted_results
+            else:
+                print(f'get task resp: {resp.status_code} - {resp.text}')
+        else:
+            print(f'Error: {resp.status_code} - {resp.text}')
+        
+        print(f"Failed to detect objects after {max_retries} attempts.")
+        return []
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/advanced_object_detector
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Object_Detector_Tool
+    tool = Advanced_Object_Detector_Tool()
+    tool.set_custom_output_dir("detected_objects")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    # print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "examples/baseball.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+
+    import json
+
+    # Execute the tool
+    try:
+        execution = tool.execute(image=image_path, labels=["baseball", "basket"], padding=20)
+        print(json.dumps(execution, indent=4))
+        print("Detected Objects:")
+        for obj in execution:
+            print(f"Detected {obj['label']} with confidence {obj['confidence score']}")
+            print(f"Bounding box: {obj['box']}")
+            print(f"Saved image (with padding): {obj['saved_image_path']}")
+            print()
+    except ValueError as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/arxiv_paper_searcher/tool.py

@@ -0,0 +1,165 @@
+import re
+import requests
+from bs4 import BeautifulSoup
+
+from octotools.tools.base import BaseTool
+
+class ArXiv_Paper_Searcher_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="ArXiv_Paper_Searcher_Tool",
+            tool_description="A tool that searches arXiv for papers based on a given query.",
+            tool_version="1.0.0",
+            input_types={
+                "query": "str - The search query for arXiv papers.",
+                "size": "int - The number of results per page (25, 50, 100, or 200). If None, use 25.",
+                "max_results": "int - The maximum number of papers to return (default: 25). Should be less than or equal to 100."
+            },
+            output_type="list - A list of dictionaries containing paper information.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(query="tool agents with large language models")',
+                    "description": "Search for papers about tool agents with large language models."
+                },
+                {
+                    "command": 'execution = tool.execute(query="quantum computing", size=100, max_results=50)',
+                    "description": "Search for quantum computing papers, with 100 results per page, returning a maximum of 50 papers."
+                },
+                {
+                    "command": 'execution = tool.execute(query="machine learning", max_results=75)',
+                    "description": "Search for machine learning papers, returning a maximum of 75 papers."
+                },
+            ],
+            user_metadata={
+                "valid_sizes": [25, 50, 100, 200],
+                "base_url": "https://arxiv.org/search/"
+            }
+        )
+
+    def build_tool(self):
+        """
+        No specific build required for this tool.
+        """
+        pass
+
+    def execute(self, query, size=None, max_results=25):
+        """
+        Executes the arXiv search tool to find papers based on the given query.
+
+        Parameters:
+            query (str): The search query for arXiv papers.
+            size (int): The number of results per page.
+            max_results (int): The maximum number of papers to return.
+
+        Returns:
+            list: A list of dictionaries containing paper information.
+        """
+        valid_sizes = self.user_metadata["valid_sizes"]
+        base_url = self.user_metadata["base_url"]
+
+        if size is None:
+            size = 25
+        elif size not in valid_sizes:
+            size = min(valid_sizes, key=lambda x: abs(x - size))
+
+        results = []
+        start = 0
+
+        max_results = min(max_results, 100) # NOTE: For traffic reasons, limit to 100 results
+
+        while len(results) < max_results:
+            params = {
+                "searchtype": "all",
+                "query": query,
+                "abstracts": "show",
+                "order": "",
+                "size": str(size),
+                "start": str(start)
+            }
+
+            try:
+                response = requests.get(base_url, params=params)
+                soup = BeautifulSoup(response.content, 'html.parser')
+
+                papers = soup.find_all("li", class_="arxiv-result")
+                if not papers:
+                    break
+
+                for paper in papers:
+                    if len(results) >= max_results:
+                        break
+
+                    title = paper.find("p", class_="title").text.strip()
+                    authors = paper.find("p", class_="authors").text.strip()
+                    authors = re.sub(r'^Authors:\s*', '', authors)
+                    authors = re.sub(r'\s+', ' ', authors).strip()
+                    
+                    abstract = paper.find("span", class_="abstract-full").text.strip()
+                    abstract = abstract.replace("△ Less", "").strip()
+                    
+                    link = paper.find("p", class_="list-title").find("a")["href"]
+
+                    results.append({
+                        "title": title,
+                        "authors": authors,
+                        "abstract": abstract,
+                        "link": f"{link}"
+                    })
+
+                start += size
+
+            except Exception as e:
+                print(f"Error searching arXiv: {e}")
+                break
+
+        return results[:max_results]
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the ArXiv_Paper_Searcher_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/arxiv_paper_searcher
+    python tool.py
+    """
+
+    import json
+
+    print("ArXiv Search Tool Test")
+
+    # Example usage of the ArXiv_Paper_Searcher_Tool
+    tool = ArXiv_Paper_Searcher_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print("Tool Metadata:")
+    print(metadata)
+
+    # Sample query for searching arXiv
+    query = "enhance mathematical reasoning with large language models"
+    # Execute the tool
+    try:
+        execution = tool.execute(query=query, size=50, max_results=10)
+        print("\n==>> Execution:")
+        print(json.dumps(execution, indent=4))  # Pretty print JSON
+        print("\n==>> Search Results:")
+        for i, paper in enumerate(execution, 1):
+            print(f"{i}. {paper['title']}")
+            print(f"   Authors: {paper['authors']}")
+            print(f"   Abstract: {paper['abstract'][:2000]}")
+            print(f"   Link: {paper['link']}")
+            print()
+    except Exception as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/base.py

@@ -0,0 +1,103 @@
+# octotools/tools/base.py
+
+from octotools.engine.openai import ChatOpenAI
+
+class BaseTool:
+    """
+    A base class for building tool classes that perform specific tasks, such as image processing or text detection.
+    """
+
+    require_llm_engine = False  # Default is False, tools that need LLM should set this to True
+
+    def __init__(self, tool_name=None, tool_description=None, tool_version=None, input_types=None, output_type=None, demo_commands=None, output_dir=None, user_metadata=None, model_string=None):
+        """
+        Initialize the base tool with optional metadata.
+
+        Parameters:
+            tool_name (str): The name of the tool.
+            tool_description (str): A description of the tool.
+            tool_version (str): The version of the tool.
+            input_types (dict): The expected input types for the tool.
+            output_type (str): The expected output type for the tool.
+            demo_commands (list): A list of example commands for using the tool.
+            output_dir (str): The directory where the tool should save its output (optional).
+            user_metadata (dict): Additional metadata specific to user needs (optional).
+            model_string (str): The model string for the LLM engine (optional, only used if require_llm_engine is True).
+        """
+        self.tool_name = tool_name
+        self.tool_description = tool_description
+        self.tool_version = tool_version
+        self.input_types = input_types
+        self.output_type = output_type
+        self.demo_commands = demo_commands
+        self.output_dir = output_dir
+        self.user_metadata = user_metadata
+        self.model_string = model_string
+
+    def set_metadata(self, tool_name, tool_description, tool_version, input_types, output_type, demo_commands, user_metadata=None):
+        """
+        Set the metadata for the tool.
+
+        Parameters:
+            tool_name (str): The name of the tool.
+            tool_description (str): A description of the tool.
+            tool_version (str): The version of the tool.
+            input_types (dict): The expected input types for the tool.
+            output_type (str): The expected output type for the tool.
+            demo_commands (list): A list of example commands for using the tool.
+            user_metadata (dict): Additional metadata specific to user needs (optional).
+        """
+        self.tool_name = tool_name
+        self.tool_description = tool_description
+        self.tool_version = tool_version
+        self.input_types = input_types
+        self.output_type = output_type
+        self.demo_commands = demo_commands
+        self.user_metadata = user_metadata
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = {
+            "tool_name": self.tool_name,
+            "tool_description": self.tool_description,
+            "tool_version": self.tool_version,
+            "input_types": self.input_types,
+            "output_type": self.output_type,
+            "demo_commands": self.demo_commands,
+            "require_llm_engine": self.require_llm_engine,
+        }
+        if self.user_metadata:
+            metadata["user_metadata"] = self.user_metadata
+        return metadata
+
+    def set_custom_output_dir(self, output_dir):
+        """
+        Set a custom output directory for the tool.
+
+        Parameters:
+            output_dir (str): The new output directory path.
+        """
+        self.output_dir = output_dir
+
+    def set_llm_engine(self, model_string):
+        """
+        Set the LLM engine for the tool.
+
+        Parameters:
+            model_string (str): The model string for the LLM engine.
+        """
+        self.model_string = model_string
+
+    def execute(self, *args, **kwargs):
+        """
+        Execute the tool's main functionality. This method should be overridden by subclasses.
+
+        Raises:
+            NotImplementedError: If the subclass does not implement this method.
+        """
+        raise NotImplementedError("Subclasses must implement the execute method.")

octotools/tools/generalist_solution_generator/tool.py

@@ -0,0 +1,144 @@
+import os
+from octotools.tools.base import BaseTool
+from octotools.engine.openai import ChatOpenAI
+
+class Generalist_Solution_Generator_Tool(BaseTool):
+    require_llm_engine = True
+    require_api_key = True
+
+    def __init__(self, model_string="gpt-4o-mini", api_key=None):
+        super().__init__(
+            tool_name="Generalist_Solution_Generator_Tool",
+            tool_description="A generalized tool that takes query from the user as prompt, and answers the question step by step to the best of its ability. It can also accept an image.",
+            tool_version="1.0.0",
+            input_types={
+                "prompt": "str - The prompt that includes query from the user to guide the agent to generate response (Examples: 'Describe this image in detail').",
+                "image": "str - The path to the image file if applicable (default: None).",
+            },
+            output_type="str - The generated response to the original query prompt",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(prompt="Summarize the following text in a few lines")',
+                    "description": "Generate a short summary given the prompt from the user."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Explain the mood of this scene.", image="path/to/image1.png")',
+                    "description": "Generate a caption focusing on the mood using a specific prompt and image."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Give your best coordinate estimate for the pacemaker in the image and return (x1, y1, x2, y2)", image="path/to/image2.png")',
+                    "description": "Generate bounding box coordinates given the image and prompt from the user. The format should be (x1, y1, x2, y2)."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Is the number of tiny objects that are behind the small metal jet less than the number of tiny things left of the tiny sedan?", image="path/to/image2.png")',
+                    "description": "Answer a question step by step given the image."
+                }
+            ],
+            # # vesion 0 (bowen) (Generalist: %; 6 Tools: %; Generalist + 6 Tools: %)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge. For optimal results: 1) Provide clear, specific prompts. 2) Use it as a starting point for complex tasks, then refine with specialized tools. 3) Verify important information from its responses. 4) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+            # vesion 2 (Generalist: 68%; 6 Tools: 66%; Generalist + 6 Tools: 54%)
+            user_metadata = {
+                "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+                "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+                "1) Provide clear, specific prompts.\n"
+                "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+                "3) For complex queries, break them down into subtasks and use the tool multiple times.\n"
+                "4) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+                "5) Verify important information from its responses.\n"
+                "6) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            }
+            # # vesion 6 (Generalist: 70%; 6 Tools: 66%; Generalist + 6 Tools: 60%)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+            #     "1) Provide clear, specific prompts.\n"
+            #     "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+            #     "3) For complex queries, break them down into smaller, focused sub-tasks and use the tool multiple times.\n"
+            #     "4) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+            #     "5) Verify important information from its responses.\n"
+            #     "6) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+            # # vesion 8 (Generalist: 68%; 6 Tools: 66%; Generalist + 6 Tools: 60%)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+            #     "1) Provide clear, specific prompts.\n"
+            #     "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+            #     "3) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+            #     "4) Verify important information from its responses.\n"
+            #     "5) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+        )
+        self.model_string = model_string  
+        self.api_key = api_key
+
+    def execute(self, prompt, image=None):
+
+        print(f"\nInitializing Generalist Tool with model: {self.model_string}")
+        multimodal = True if image else False
+        llm_engine = ChatOpenAI(model_string=self.model_string, is_multimodal=multimodal, api_key=self.api_key)
+
+        try:
+            input_data = [prompt]
+            if multimodal:
+                if not os.path.isfile(image):
+                    return "Error: Invalid image file path."
+                try:
+                    with open(image, 'rb') as file:
+                        image_bytes = file.read()
+                    input_data.append(image_bytes)
+                except Exception as e:
+                    return f"Error reading image file: {str(e)}"
+
+                response = llm_engine(input_data)
+            else:
+                response = llm_engine(input_data[0])
+            return response
+        except Exception as e:
+            return f"Error generating response: {str(e)}"
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools
+    python tools/default/tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    print(f"Script directory: {script_dir}")
+
+    # Example usage of the Generalist_Tool
+    tool = Generalist_Solution_Generator_Tool()
+    # tool = Generalist_Solution_Generator_Tool(model_string="gpt-4o-mini")
+    # tool = Generalist_Solution_Generator_Tool(model_string="gpt-4o")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "../../tasks/minitoolbench/data/mathvista_113.png"
+    relative_image_path = "examples/mathvista_113.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+    prompt = "Describe the image in detail."
+
+    # Execute the tool with default prompt
+    try:
+        execution = tool.execute(prompt=prompt, image=image_path)
+        # execution = tool.execute(prompt=prompt)
+        print("Generated Response:")
+        print(execution)
+    except Exception as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/google_search/tool.py

@@ -0,0 +1,136 @@
+import os
+import requests
+from typing import List, Dict, Any
+
+from octotools.tools.base import BaseTool
+
+from dotenv import load_dotenv
+load_dotenv()
+
+class Google_Search_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Google_Search_Tool",
+            tool_description="A tool that performs Google searches based on a given text query.",
+            tool_version="1.0.0",
+            input_types={
+                "query": "str - The search query to be used for the Google search.",
+                "num_results": "int - The number of search results to return (default: 10).",
+            },
+            output_type="list - A list of dictionaries containing search result information.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(query="Python programming")',
+                    "description": "Perform a Google search for 'Python programming' and return the default number of results."
+                },
+                {
+                    "command": 'execution = tool.execute(query="Machine learning tutorials", num_results=5)',
+                    "description": "Perform a Google search for 'Machine learning tutorials' and return 5 results."
+                },
+            ],
+        )
+        # self.api_key = os.getenv("GOOGLE_API_KEY")
+        self.api_key = os.getenv("GOOGLE_API_KEY") # NOTE: Replace with your own API key (Ref: https://developers.google.com/custom-search/v1/introduction)
+        self.cx = os.getenv("GOOGLE_CX") # NOTE: Replace with your own custom search (Ref: https://programmablesearchengine.google.com/controlpanel/all)
+        self.base_url = "https://www.googleapis.com/customsearch/v1"
+
+    def google_search(self, query: str, num_results: int = 10) -> Dict[str, Any]:
+        """
+        Performs a Google search using the provided query.
+
+        Parameters:
+            query (str): The search query.
+            num_results (int): The number of search results to return.
+
+        Returns:
+            Dict[str, Any]: The raw search results from the Google API.
+        """
+        params = {
+            'q': query,
+            'key': self.api_key,
+            'cx': self.cx,
+            'num': num_results
+        }
+        
+        response = requests.get(self.base_url, params=params)
+        return response.json()
+
+    def execute(self, query: str, num_results: int = 10) -> List[Dict[str, Any]]:
+        """
+        Executes a Google search based on the provided query.
+
+        Parameters:
+            query (str): The search query.
+            num_results (int): The number of search results to return (default: 10).
+
+        Returns:
+            List[Dict[str, Any]]: A list of dictionaries containing search result information.
+        """
+        if not self.api_key:
+            return [{"error": "Google API key is not set. Please set the GOOGLE_API_KEY environment variable."}]
+
+        try:
+            results = self.google_search(query, num_results)
+            print(results)
+            
+            if 'items' in results:
+                return [
+                    {
+                        "title": item['title'],
+                        "link": item['link'],
+                        "snippet": item['snippet']
+                    }
+                    for item in results['items']
+                ]
+            else:
+                return [{"error": "No results found."}]
+        except Exception as e:
+            return [{"error": f"An error occurred: {str(e)}"}]
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the Google_Search_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+
+    export GOOGLE_API_KEY=your_api_key_here
+    cd octotools/tools/google_search
+    python tool.py
+    """
+
+    # Example usage of the Google_Search_Tool
+    tool = Google_Search_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Execute the tool to perform a Google search
+    query = "nobel prize winners in chemistry 2024"
+    try:
+        execution = tool.execute(query=query, num_results=5)
+        print("\nExecution Result:")
+        print(f"Search query: {query}")
+        print(f"Number of results: {len(execution)}")
+        print("\nSearch Results:")
+        if "error" in execution[0]:
+            print(f"Error: {execution[0]['error']}")
+        else:
+            for i, item in enumerate(execution, 1):
+                print(f"\n{i}. Title: {item['title']}")
+                print(f"   URL: {item['link']}")
+                print(f"   Snippet: {item['snippet']}")
+    except Exception as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/image_captioner/tool.py

@@ -0,0 +1,96 @@
+import os
+from octotools.tools.base import BaseTool
+from octotools.engine.openai import ChatOpenAI
+
+class Image_Captioner_Tool(BaseTool):
+    require_llm_engine = True
+
+    def __init__(self, model_string="gpt-4o-mini"):
+        super().__init__(
+            tool_name="Image_Captioner_Tool",
+            tool_description="A tool that generates captions for images using OpenAI's multimodal model.",
+            tool_version="1.0.0",
+            input_types={
+                "image": "str - The path to the image file.",
+                "prompt": "str - The prompt to guide the image captioning (default: 'Describe this image in detail.').",
+            },
+            output_type="str - The generated caption for the image.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png")',
+                    "description": "Generate a caption for an image using the default prompt and model."
+                },
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", prompt="Explain the mood of this scene.")',
+                    "description": "Generate a caption focusing on the mood using a specific prompt and model."
+                }
+            ],
+            user_metadata = {
+                "limitation": "The Image_Captioner_Tool provides general image descriptions but has limitations: 1) May make mistakes in complex scenes, counting, attribute detection, and understanding object relationships. 2) Might not generate comprehensive captions, especially for images with multiple objects or abstract concepts. 3) Performance varies with image complexity. 4) Struggles with culturally specific or domain-specific content. 5) May overlook details or misinterpret object relationships. For precise descriptions, consider: using it with other tools for context/verification, as an initial step before refinement, or in multi-step processes for ambiguity resolution. Verify critical information with specialized tools or human expertise when necessary."
+            },
+        )
+        print(f"\nInitializing Image Captioner Tool with model: {model_string}")
+        self.llm_engine = ChatOpenAI(model_string=model_string, is_multimodal=True) if model_string else None
+
+    def execute(self, image, prompt="Describe this image in detail."):
+        try:
+            if not self.llm_engine:
+                return "Error: LLM engine not initialized. Please provide a valid model_string."
+                
+            input_data = [prompt]
+            
+            if image and os.path.isfile(image):
+                try:
+                    with open(image, 'rb') as file:
+                        image_bytes = file.read()
+                    input_data.append(image_bytes)
+                except Exception as e:
+                    return f"Error reading image file: {str(e)}"
+            else:
+                return "Error: Invalid image file path."
+
+            caption = self.llm_engine(input_data)
+            return caption
+        except Exception as e:
+            return f"Error generating caption: {str(e)}"
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        metadata['require_llm_engine'] = self.require_llm_engine # NOTE: can be removed if not needed
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/image_captioner
+    python tool.py
+    """
+
+    import json
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Image_Captioner_Tool
+    # tool = Image_Captioner_Tool()
+    tool = Image_Captioner_Tool(model_string="gpt-4o")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "examples/baseball.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+
+    # Execute the tool with default prompt
+    try:
+        execution = tool.execute(image=image_path)
+        print("Generated Caption:")
+        print(json.dumps(execution, indent=4)) 
+    except Exception as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/nature_news_fetcher/tool.py

@@ -0,0 +1,181 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+import time
+
+from octotools.tools.base import BaseTool
+
+class Nature_News_Fetcher_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Nature_News_Fetcher_Tool",
+            tool_description="A tool that fetches the latest news articles from Nature.",
+            tool_version="1.0.0",
+            input_types={
+                "num_articles": "int - The number of articles to fetch (default: 100).",
+                "max_pages": "int - The maximum number of pages to fetch (default: 5).",
+            },
+            output_type="list - A list of dictionaries containing information about the latest Nature news articles.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute()',
+                    "description": "Fetch the latest 100 news articles from Nature."
+                },
+                {
+                    "command": 'execution = tool.execute(num_articles=50, max_pages=3)',
+                    "description": "Fetch the latest 50 news articles from Nature, searching up to 3 pages."
+                },
+            ],
+        )
+        self.base_url = "https://www.nature.com/nature/articles"
+
+    def fetch_page(self, page_number):
+        """
+        Fetches a single page of news articles from Nature's website.
+
+        Parameters:
+            page_number (int): The page number to fetch.
+
+        Returns:
+            str: The HTML content of the page.
+        """
+        params = {
+            "searchType": "journalSearch",
+            "sort": "PubDate",
+            "type": "news",
+            "page": str(page_number)
+        }
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+        }
+        response = requests.get(self.base_url, params=params, headers=headers)
+        response.raise_for_status()
+        return response.text
+
+    def parse_articles(self, html_content):
+        """
+        Parses the HTML content and extracts article information.
+
+        Parameters:
+            html_content (str): The HTML content of the page.
+
+        Returns:
+            list: A list of dictionaries containing article information.
+        """
+        soup = BeautifulSoup(html_content, 'html.parser')
+        articles_section = soup.find('section', id='new-article-list')
+        if not articles_section:
+            return []
+
+        articles = []
+        for article in articles_section.find_all('article', class_='c-card'):
+            title_elem = article.find('h3', class_='c-card__title')
+            title = title_elem.text.strip() if title_elem else "No title found"
+            
+            url_elem = title_elem.find('a') if title_elem else None
+            url = "https://www.nature.com" + url_elem['href'] if url_elem and 'href' in url_elem.attrs else "No URL found"
+            
+            description_elem = article.find('div', {'data-test': 'article-description'})
+            description = description_elem.text.strip() if description_elem else "No description available"
+            
+            authors_elem = article.find('ul', {'data-test': 'author-list'})
+            authors = [author.text.strip() for author in authors_elem.find_all('li')] if authors_elem else ["No authors found"]
+            
+            date_elem = article.find('time')
+            date = date_elem['datetime'] if date_elem and 'datetime' in date_elem.attrs else "No date found"
+
+            image_elem = article.find('img')
+            image_url = image_elem['src'] if image_elem and 'src' in image_elem.attrs else "No image found"
+
+            articles.append({
+                'title': title,
+                'url': url,
+                'description': description,
+                'authors': authors,
+                'date': date,
+                'image_url': image_url
+            })
+
+        return articles
+
+    def execute(self, num_articles=100, max_pages=5):
+        """
+        Fetches the latest news articles from Nature's website.
+
+        Parameters:
+            num_articles (int): The number of articles to fetch.
+            max_pages (int): The maximum number of pages to fetch.
+
+        Returns:
+            list: A list of dictionaries containing article information.
+        """
+        all_articles = []
+        page_number = 1
+
+        try:
+            while len(all_articles) < num_articles and page_number <= max_pages:
+                html_content = self.fetch_page(page_number)
+                page_articles = self.parse_articles(html_content)
+                
+                if not page_articles:
+                    break  # No more articles found
+
+                all_articles.extend(page_articles)
+                page_number += 1
+                time.sleep(1)  # Be polite to the server
+
+            return all_articles[:num_articles]
+        except Exception as e:
+            return [{"error": str(e)}]
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the Nature_News_Fetcher_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+
+    cd octotools/tools/nature_news_fetcher
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Nature_News_Fetcher_Tool
+    tool = Nature_News_Fetcher_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    import json
+  
+
+    # Execute the tool to fetch the latest 10 articles (for demonstration purposes)
+    try:
+        execution = tool.execute(num_articles=10, max_pages=1)
+        print(json.dumps(execution, indent=4))
+        print("\nExecution Result:")
+        print(f"Number of articles fetched: {len(execution)}")
+        print("\nSample articles:")
+        for i, article in enumerate(execution[:10], 1):
+            print(f"\n{i}. Title: {article['title']}")
+            print(f"   URL: {article['url']}")
+            print(f"   Description: {article['description'][:100]}...")  # Show first 100 characters
+            print(f"   Authors: {', '.join(article['authors'])}")
+            print(f"   Date: {article['date']}")
+            print(f"   Image URL: {article['image_url']}")
+    except Exception as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/object_detector/tool.py

@@ -0,0 +1,179 @@
+# Grounding DINO Object Detection Tool
+# https://huggingface.co/IDEA-Research/grounding-dino
+
+import os
+import time
+import torch
+from transformers import pipeline
+
+from octotools.tools.base import BaseTool
+from PIL import Image, ImageOps
+
+import os
+# If CUDA_HOME is set, print the value
+print(os.environ.get('CUDA_HOME', 'CUDA_HOME is not set'))
+
+# Suppress stderr by redirecting it to /dev/null
+import sys
+sys.stderr = open(os.devnull, 'w')
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+class Object_Detector_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Object_Detector_Tool",
+            tool_description="A tool that detects objects in an image using the Grounding DINO model and saves individual object images with empty padding.",
+            tool_version="1.0.0",
+            input_types={
+                "image": "str - The path to the image file.",
+                "labels": "list - A list of object labels to detect.",
+                "threshold": "float - The confidence threshold for detection (default: 0.35).",
+                "model_size": "str - The size of the model to use ('tiny' or 'base', default: 'tiny').",
+                "padding": "int - The number of pixels to add as empty padding around detected objects (default: 20)."
+            },
+            output_type="list - A list of detected objects with their scores, bounding boxes, and saved image paths.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", labels=["baseball", "basket"])',
+                    "description": "Detect baseball and basket in an image, save the detected objects with default empty padding, and return their paths."
+                },
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", labels=["car", "person"], threshold=0.5, model_size="base", padding=15)',
+                    "description": "Detect car and person in an image using the base model, save the detected objects with 15 pixels of empty padding, and return their paths."
+                }
+            ],
+            user_metadata={
+                "limitation": "The model may not always detect objects accurately, and its performance can vary depending on the input image and the associated labels. It typically struggles with detecting small objects, objects that are uncommon, or objects with limited or specific attributes. For improved accuracy or better detection in certain situations, consider using supplementary tools or image processing techniques to provide additional information for verification."
+            }
+        )
+
+    def preprocess_caption(self, caption):
+        result = caption.lower().strip()
+        if result.endswith("."):
+            return result
+        return result + "."
+
+    def build_tool(self, model_size='tiny'):
+        model_name = f"IDEA-Research/grounding-dino-{model_size}"
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        try:
+            pipe = pipeline(model=model_name, task="zero-shot-object-detection", device=device)
+            return pipe
+        except Exception as e:
+            print(f"Error building the Object Detection tool: {e}")
+            return None
+
+    def save_detected_object(self, image, box, image_name, label, index, padding):
+        object_image = image.crop(box)
+        padded_image = ImageOps.expand(object_image, border=padding, fill='white')
+        
+        filename = f"{image_name}_{label}_{index}.png"
+        os.makedirs(self.output_dir, exist_ok=True)
+        save_path = os.path.join(self.output_dir, filename)
+        
+        padded_image.save(save_path)
+        return save_path
+
+    def execute(self, image, labels, threshold=0.35, model_size='tiny', padding=20, max_retries=10, retry_delay=5, clear_cuda_cache=False):
+        for attempt in range(max_retries):
+            try:
+                saved_files = []
+
+                pipe = self.build_tool(model_size)
+                if pipe is None:
+                    raise ValueError("Failed to build the Object Detection tool.")
+                
+                preprocessed_labels = [self.preprocess_caption(label) for label in labels]
+                results = pipe(image, candidate_labels=preprocessed_labels, threshold=threshold)
+                
+                formatted_results = []
+                original_image = Image.open(image)
+                image_name = os.path.splitext(os.path.basename(image))[0]
+                
+                object_counts = {}
+
+                for result in results:
+                    box = tuple(result["box"].values())
+                    label = result["label"]
+                    score = round(result["score"], 2)
+                    if label.endswith("."):
+                        label = label[:-1]
+                    
+                    object_counts[label] = object_counts.get(label, 0) + 1
+                    index = object_counts[label]
+                    
+                    save_path = self.save_detected_object(original_image, box, image_name, label, index, padding)
+            
+                    formatted_results.append({
+                        "label": label,
+                        "confidence score": score,
+                        "box": box,
+                        "saved_image_path": save_path
+                    })
+
+                return formatted_results
+            
+            except RuntimeError as e:
+                if "CUDA out of memory" in str(e):
+                    print(f"CUDA out of memory error on attempt {attempt + 1}.")
+                    if clear_cuda_cache:
+                        print("Clearing CUDA cache and retrying...")
+                        torch.cuda.empty_cache()
+                    else:
+                        print(f"Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                    continue
+                else:
+                    print(f"Runtime error: {e}")
+                    break
+            except Exception as e:
+                print(f"Error detecting objects: {e}")
+                break
+        
+        print(f"Failed to detect objects after {max_retries} attempts.")
+        return []
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/object_detector
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Object_Detector_Tool
+    tool = Object_Detector_Tool()
+    tool.set_custom_output_dir("detected_objects")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "examples/baseball.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+
+    # Execute the tool
+    try:
+        execution = tool.execute(image=image_path, labels=["baseball", "basket"], padding=20)
+        print("Detected Objects:")
+        for obj in execution:
+            print(f"Detected {obj['label']} with confidence {obj['confidence score']}")
+            print(f"Bounding box: {obj['box']}")
+            print(f"Saved image (with padding): {obj['saved_image_path']}")
+            print()
+    except ValueError as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/pubmed_search/tool.py

@@ -0,0 +1,112 @@
+import os
+import json
+from pymed import PubMed
+from metapub import PubMedFetcher
+from octotools.tools.base import BaseTool
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+# Suppress stderr by redirecting it to /dev/null
+import sys
+sys.stderr = open(os.devnull, 'w')
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+class Pubmed_Search_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Pubmed_Search_Tool",
+            tool_description="A tool that searches PubMed Central to retrieve relevant article abstracts based on a given list of text queries. Use this ONLY if you cannot use the other more specific ontology tools.",
+            tool_version="1.0.0",
+            input_types={
+                "queries": "list[str] - list of queries terms for searching PubMed."
+            },
+            output_type="list - List of items matching the search query. Each item consists of the title, abstract, keywords, and URL of the article. If no results found, a string message is returned.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(queries=["scoliosis", "injury"])',
+                    "description": "Search for PubMed articles mentioning 'scoliosis' OR 'injury'."
+                },
+                {
+                    "command": 'execution = tool.execute(queries=["COVID", "vaccine", "occupational health"])',
+                    "description": "Search for PubMed articles mentioning 'COVID' OR 'vaccine' OR 'occupational health'."
+                }
+            ],
+            user_metadata={
+                'limitations': "Try to use shorter and more general search queries."
+            }
+        )
+        self.pubmed = PubMed(tool="MyTool", email="[email protected]")
+        self.fetch = PubMedFetcher()
+
+    @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(3))
+    def search_query(self, query_str, max_results=10):
+        return self.pubmed.query(query_str, max_results=max_results)
+
+    def execute(self, queries, max_results=10):
+        try:
+            query_str = f"({'[Title/Abstract] OR '.join(queries) + '[Title/Abstract]'}) AND hasabstract[All Fields] AND fha[Filter]"
+            max_results = min(max_results, 50)
+
+            results = self.search_query(query_str, max_results=max_results) # API can only get most recent
+
+            items = []
+            for article in results:
+                try:
+                    article = json.loads(article.toJSON())
+                    pubmed_id = article['pubmed_id'] # get id using pymed then get content using metapub
+
+                    article = self.fetch.article_by_pmid(pubmed_id)
+                    items.append({
+                        'title': article.title,
+                        'abstract': article.abstract,
+                        'keywords': article.keywords,
+                        'url': article.url
+                    })
+                except:
+                    continue
+
+            if len(items) == 0:
+                return "No results found for search query. Try another query or tool."
+            
+            return items
+        
+        except Exception as e:
+            print(f"Error searching PubMed: {e}")
+            return []
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/pubmed_search
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage
+    tool = Pubmed_Search_Tool()
+
+    # Queries
+    queries = ["COVID occupational health"]
+
+    # Execute the tool
+    try:
+        execution = tool.execute(queries=queries)
+        print(execution)
+    except ValueError as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/python_code_generator/tool.py

@@ -0,0 +1,243 @@
+# octotools/tools/python_code_generator/tool.py
+
+import os
+import re
+import sys
+from io import StringIO
+import contextlib
+
+
+from octotools.tools.base import BaseTool
+from octotools.engine.openai import ChatOpenAI
+
+import signal
+from contextlib import contextmanager
+
+# Custom exception for code execution timeout
+class TimeoutException(Exception):
+    pass
+
+# Custom context manager for code execution timeout
+@contextmanager
+def timeout(seconds):
+    def timeout_handler(signum, frame):
+        raise TimeoutException("Code execution timed out")
+
+    # Set the timeout handler
+    original_handler = signal.signal(signal.SIGALRM, timeout_handler)
+    signal.alarm(seconds)
+    
+    try:
+        yield
+    finally:
+        # Restore the original handler and disable the alarm
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, original_handler)
+
+
+class Python_Code_Generator_Tool(BaseTool):
+    require_llm_engine = True
+
+    def __init__(self, model_string="gpt-4o-mini"):
+        super().__init__(
+            tool_name="Python_Code_Generator_Tool",
+            tool_description="A tool that generates and executes simple Python code snippets for basic arithmetical calculations and math-related problems. The generated code runs in a highly restricted environment with only basic mathematical operations available.",
+            tool_version="1.0.0",
+            input_types={
+                "query": "str - A clear, specific description of the arithmetic calculation or math problem to be solved, including any necessary numerical inputs."},
+            output_type="dict - A dictionary containing the generated code, calculation result, and any error messages.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(query="Calculate the factorial of 5")',
+                    "description": "Generate a Python code snippet to calculate the factorial of 5."
+                },
+                {
+                    "command": 'execution = tool.execute(query="Find the sum of prime numbers up to 50")',
+                    "description": "Generate a Python code snippet to find the sum of prime numbers up to 50."
+                },
+                {
+                    "command": 'query="Given the list [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], calculate the sum of squares of odd numbers"\nexecution = tool.execute(query=query)',
+                    "description": "Generate a Python function for a specific mathematical operation on a given list of numbers."
+                },
+            ],
+            user_metadata = {
+                "limitations": [
+                    "Restricted to basic Python arithmetic operations and built-in mathematical functions.",
+                    "Cannot use any external libraries or modules, including those in the Python standard library.",
+                    "Limited to simple mathematical calculations and problems.",
+                    "Cannot perform any string processing, data structure manipulation, or complex algorithms.",
+                    "No access to any system resources, file operations, or network requests.",
+                    "Cannot use 'import' statements.",
+                    "All calculations must be self-contained within a single function or script.",
+                    "Input must be provided directly in the query string.",
+                    "Output is limited to numerical results or simple lists/tuples of numbers."
+                ],
+                "best_practices": [
+                    "Provide clear and specific queries that describe the desired mathematical calculation.",
+                    "Include all necessary numerical inputs directly in the query string.",
+                    "Keep tasks focused on basic arithmetic, algebraic calculations, or simple mathematical algorithms.",
+                    "Ensure all required numerical data is included in the query.",
+                    "Verify that the query only involves mathematical operations and does not require any data processing or complex algorithms.",
+                    "Review generated code to ensure it only uses basic Python arithmetic operations and built-in math functions."
+                ]
+            }
+        )
+        print(f"\nInitializing Python_Code_Generator_Tool with model_string: {model_string}")
+        self.llm_engine = ChatOpenAI(model_string=model_string, is_multimodal=False) if model_string else None
+
+    @staticmethod
+    def preprocess_code(code):
+        """
+        Preprocesses the generated code snippet by extracting it from the response.
+
+        Parameters:
+            code (str): The response containing the code snippet.
+
+        Returns:
+            str: The extracted code snippet.
+        """
+        code = re.search(r"```python(.*)```", code, re.DOTALL).group(1).strip()
+        return code
+
+    @contextlib.contextmanager
+    def capture_output(self):
+        """
+        Context manager to capture the standard output.
+
+        Yields:
+            StringIO: The captured output.
+        """
+        new_out = StringIO()
+        old_out = sys.stdout
+        sys.stdout = new_out
+        try:
+            yield sys.stdout
+        finally:
+            sys.stdout = old_out
+
+    def execute_code_snippet(self, code):
+        """
+        Executes the given Python code snippet.
+
+        Parameters:
+            code (str): The Python code snippet to be executed.
+
+        Returns:
+            dict: A dictionary containing the printed output and local variables.
+        """
+        # Check for dangerous functions and remove them
+        dangerous_functions = ['exit', 'quit', 'sys.exit']
+        for func in dangerous_functions:
+            if func in code:
+                print(f"Warning: Removing unsafe '{func}' call from code")
+                # Use regex to remove function calls with any arguments
+                code = re.sub(rf'{func}\s*\([^)]*\)', 'break', code)
+
+        try:
+            execution_code = self.preprocess_code(code)
+
+            # Execute with 10-second timeout
+            with timeout(10):
+                try:
+                    exec(execution_code)
+                except TimeoutException:
+                    print("Error: Code execution exceeded 60 seconds timeout")
+                    return {"error": "Execution timed out after 60 seconds"}
+                except Exception as e:
+                    print(f"Error executing code: {e}")
+                    return {"error": str(e)}
+                
+            # Capture the output and local variables
+            local_vars = {}
+            with self.capture_output() as output:
+                exec(execution_code, {}, local_vars)
+            printed_output = output.getvalue().strip()
+
+            # Filter out built-in variables and modules
+            """
+            only the variables used in the code are returned, 
+            excluding built-in variables (which start with '__') and imported modules.
+            """
+            used_vars = {k: v for k, v in local_vars.items() 
+                         if not k.startswith('__') and not isinstance(v, type(sys))}
+            
+            return {"printed_output": printed_output, "variables": used_vars}
+        
+        except Exception as e:
+            print(f"Error executing code: {e}")
+            return {"error": str(e)}
+
+    def execute(self, query):
+        """
+        Generates and executes Python code based on the provided query.
+
+        Parameters:
+            query (str): A query describing the desired operation.
+
+        Returns:
+            dict: A dictionary containing the executed output, local variables, or any error message.
+        """
+
+        if not self.llm_engine:
+            raise ValueError("LLM engine not initialized. Please provide a valid model_string when initializing the tool.")
+
+        task_description = """
+        Given a query, generate a Python code snippet that performs the specified operation on the provided data. Please think step by step. Ensure to break down the process into clear, logical steps. Make sure to print the final result in the generated code snippet with a descriptive message explaining what the output represents. The final output should be presented in the following format:
+
+        ```python
+        <code snippet>
+        ```
+        """
+        task_description = task_description.strip()
+        full_prompt = f"Task:\n{task_description}\n\nQuery:\n{query}"
+
+        response = self.llm_engine(full_prompt)
+        result_or_error = self.execute_code_snippet(response)
+        return result_or_error
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the Python_Code_Generator_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        metadata["require_llm_engine"] = self.require_llm_engine # NOTE: can be removed if not needed
+        return metadata
+
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/python_code_generator
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Python_Code_Generator_Tool
+    tool = Python_Code_Generator_Tool()
+    tool = Python_Code_Generator_Tool(model_string="gpt-4o-mini")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Sample query for generating and executing Python code
+    queries = [
+        "Given the number list: [1, 2, 3, 4, 5], calculate the sum of all the numbers in the list.",
+    ]
+    for query in queries:
+        print(f"\n###Query: {query}")
+        # Execute the tool with the sample query
+        try:
+            execution = tool.execute(query=query)
+            print("\n###Execution Result:", execution)
+        except ValueError as e:
+            print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/relevant_patch_zoomer/tool.py

@@ -0,0 +1,188 @@
+import os
+import cv2
+from pydantic import BaseModel
+from octotools.tools.base import BaseTool
+from octotools.engine.openai import ChatOpenAI
+
+class PatchZoomerResponse(BaseModel):
+    analysis: str
+    patch: list[str]
+
+class Relevant_Patch_Zoomer_Tool(BaseTool):
+    require_llm_engine = True
+
+    def __init__(self, model_string="gpt-4o"):
+        super().__init__(
+            tool_name="Relevant_Patch_Zoomer_Tool",
+            tool_description="A tool that analyzes an image, divides it into 5 regions (4 quarters + center), and identifies the most relevant patches based on a question. The returned patches are zoomed in by a factor of 2.",
+            tool_version="1.0.0",
+            input_types={
+                "image": "str - The path to the image file.",
+                "question": "str - The question about the image content.",
+            },
+            output_type="dict - Contains analysis text and list of saved zoomed patch paths.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.jpg", question="What is the color of the car?")',
+                    "description": "Analyze image and return relevant zoomed patches that show the car's color."
+                }
+            ],
+            user_metadata = {
+                "best_practices": [
+                    "It might be helpful to zoom in on the image first to get a better look at the object(s).",
+                    "It might be helpful if the question requires a close-up view of the object(s), symbols, texts, etc.",
+                    "The tool should be used to provide a high-level analysis first, and then use other tools for fine-grained analysis. For example, you can use Relevant_Patch_Zoomer_Tool first to get a zoomed patch of specific objects, and then use Image_Captioner_Tool to describe the objects in detail."
+                ]
+            }
+        )
+        self.matching_dict = {
+            "A": "top-left",
+            "B": "top-right",
+            "C": "bottom-left",
+            "D": "bottom-right",
+            "E": "center"
+        }
+
+        print(f"\nInitializing Patch Zoomer Tool with model: {model_string}")
+        self.llm_engine = ChatOpenAI(model_string=model_string, is_multimodal=True) if model_string else None
+        
+    def _save_patch(self, image_path, patch, save_path, zoom_factor=2):
+        """Extract and save a specific patch from the image with 10% margins."""
+        img = cv2.imread(image_path)
+        height, width = img.shape[:2]
+        
+        quarter_h = height // 2
+        quarter_w = width // 2
+        
+        margin_h = int(quarter_h * 0.1)
+        margin_w = int(quarter_w * 0.1)
+        
+        patch_coords = {
+            'A': ((max(0, 0 - margin_w), max(0, 0 - margin_h)),
+                  (min(width, quarter_w + margin_w), min(height, quarter_h + margin_h))),
+            'B': ((max(0, quarter_w - margin_w), max(0, 0 - margin_h)),
+                  (min(width, width + margin_w), min(height, quarter_h + margin_h))),
+            'C': ((max(0, 0 - margin_w), max(0, quarter_h - margin_h)),
+                  (min(width, quarter_w + margin_w), min(height, height + margin_h))),
+            'D': ((max(0, quarter_w - margin_w), max(0, quarter_h - margin_h)),
+                  (min(width, width + margin_w), min(height, height + margin_h))),
+            'E': ((max(0, quarter_w//2 - margin_w), max(0, quarter_h//2 - margin_h)),
+                  (min(width, quarter_w//2 + quarter_w + margin_w), 
+                   min(height, quarter_h//2 + quarter_h + margin_h)))
+        }
+        
+        (x1, y1), (x2, y2) = patch_coords[patch]
+        patch_img = img[y1:y2, x1:x2]
+        
+        zoomed_patch = cv2.resize(patch_img, 
+                                (patch_img.shape[1] * zoom_factor, 
+                                 patch_img.shape[0] * zoom_factor), 
+                                interpolation=cv2.INTER_LINEAR)
+        
+        os.makedirs(os.path.dirname(save_path), exist_ok=True)
+        cv2.imwrite(save_path, zoomed_patch)
+        return save_path
+
+    def execute(self, image, question, zoom_factor=2):
+        try:
+            if not self.llm_engine:
+                return "Error: LLM engine not initialized. Please provide a valid model_string."
+            
+            # Prepare the prompt
+            prompt = f"""
+Analyze this image to identify the most relevant region(s) for answering the question:
+
+Question: {question}
+
+The image is divided into 5 regions:
+- (A) Top-left quarter
+- (B) Top-right quarter
+- (C) Bottom-left quarter
+- (D) Bottom-right quarter
+- (E) Center region (1/4 size, overlapping middle section)
+
+Instructions:
+1. First describe what you see in each of the five regions.
+2. Then select the most relevant region(s) to answer the question.
+3. Choose only the minimum necessary regions - avoid selecting redundant areas that show the same content. For example, if one patch contains the entire object(s), do not select another patch that only shows a part of the same object(s).
+
+
+Response format:
+<analysis>: Describe the image and five patches first. Then analyze the question and select the most relevant patch or list of patches.
+<patch>: List of letters (A-E)
+"""
+            # Read image and create input data
+            with open(image, 'rb') as file:
+                image_bytes = file.read()
+            input_data = [prompt, image_bytes]
+            
+            # Get response from LLM
+            response = self.llm_engine(input_data, response_format=PatchZoomerResponse)
+            
+            # Save patches
+            image_dir = os.path.dirname(image)
+            image_name = os.path.splitext(os.path.basename(image))[0]
+            
+            # Update the return structure
+            patch_info = []
+            for patch in response.patch:
+                patch_name = self.matching_dict[patch]
+                save_path = os.path.join(self.output_dir, 
+                                       f"{image_name}_{patch_name}_zoomed_{zoom_factor}x.png")
+                saved_path = self._save_patch(image, patch, save_path, zoom_factor)
+                save_path = os.path.abspath(saved_path)
+                patch_info.append({
+                    "path": save_path,
+                    "description": f"The {self.matching_dict[patch]} region of the image: {image}."
+                })
+            
+            return {
+                "analysis": response.analysis,
+                "patches": patch_info
+            }
+            
+        except Exception as e:
+            print(f"Error in patch zooming: {e}")
+            return None
+
+    def get_metadata(self):
+        return super().get_metadata()
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/relevant_patch_zoomer
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Relevant_Patch_Zoomer_Tool
+    tool = Relevant_Patch_Zoomer_Tool()
+    tool.set_custom_output_dir(f"{script_dir}/zoomed_patches")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "examples/car.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+    question = "What is the color of the car?"
+
+    # Execute the tool
+    try:
+        result = tool.execute(image=image_path, question=question)
+        if result:
+            print("\nDetected Patches:")
+            for patch in result['patches']:
+                print(f"Path: {patch['path']}")
+                print(f"Description: {patch['description']}")
+                print()
+    except Exception as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/text_detector/tool.py

@@ -0,0 +1,173 @@
+# octotools/tools/text_detector/tool.py
+
+import os
+import time
+from octotools.tools.base import BaseTool
+
+import warnings
+warnings.filterwarnings("ignore")
+
+class Text_Detector_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Text_Detector_Tool",
+            tool_description="A tool that detects text in an image using EasyOCR.",
+            tool_version="1.0.0",
+            input_types={
+                "image": "str - The path to the image file.",
+                "languages": "list - A list of language codes for the OCR model.",
+                "detail": "int - The level of detail in the output. Set to 0 for simpler output, 1 for detailed output."
+            },
+            output_type="list - A list of detected text blocks.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", languages=["en"])',
+                    "description": "Detect text in an image using the default language (English)."
+                },
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", languages=["en", "de"])',
+                    "description": "Detect text in an image using multiple languages (English and German)."
+                },
+                {
+                    "command": 'execution = tool.execute(image="path/to/image.png", languages=["en"], detail=0)',
+                    "description": "Detect text in an image with simpler output (text without coordinates and scores)."
+                },
+            ],
+            user_metadata={
+                "frequently_used_language": {
+                    "ch_sim": "Simplified Chinese",
+                    "ch_tra": "Traditional Chinese",
+                    "de": "German",
+                    "en": "English",
+                    "es": "Spanish",
+                    "fr": "French",
+                    "hi": "Hindi",
+                    "ja": "Japanese",
+                }
+            }
+        )
+
+    def build_tool(self, languages=None):
+        """
+        Builds and returns the EasyOCR reader model.
+
+        Parameters:
+            languages (list): A list of language codes for the OCR model.
+
+        Returns:
+            easyocr.Reader: An initialized EasyOCR Reader object.
+        """
+        languages = languages or ["en"]  # Default to English if no languages provided
+        try:
+            import easyocr
+            reader = easyocr.Reader(languages)
+            return reader
+        except ImportError:
+            raise ImportError("Please install the EasyOCR package using 'pip install easyocr'.")
+        except Exception as e:
+            print(f"Error building the OCR tool: {e}")
+            return None
+    
+    def execute(self, image, languages=None, max_retries=10, retry_delay=5, clear_cuda_cache=False, **kwargs):
+        """
+        Executes the OCR tool to detect text in the provided image.
+
+        Parameters:
+            image (str): The path to the image file.
+            languages (list): A list of language codes for the OCR model.
+            max_retries (int): Maximum number of retry attempts.
+            retry_delay (int): Delay in seconds between retry attempts.
+            clear_cuda_cache (bool): Whether to clear CUDA cache on out-of-memory errors.
+            **kwargs: Additional keyword arguments for the OCR reader.
+
+        Returns:
+            list: A list of detected text blocks.
+        """
+        languages = languages or ["en"]
+
+        for attempt in range(max_retries):
+            try:
+                reader = self.build_tool(languages)
+                if reader is None:
+                    raise ValueError("Failed to build the OCR tool.")
+                
+                result = reader.readtext(image, **kwargs)
+                try:
+                    # detail = 1: Convert numpy types to standard Python types
+                    cleaned_result = [
+                        ([[int(coord[0]), int(coord[1])] for coord in item[0]], item[1], round(float(item[2]), 2))
+                        for item in result
+                    ]
+                    return cleaned_result
+                except Exception as e:
+                    # detail = 0
+                    return result
+
+            except RuntimeError as e:
+                if "CUDA out of memory" in str(e):
+                    print(f"CUDA out of memory error on attempt {attempt + 1}.")
+                    if clear_cuda_cache:
+                        print("Clearing CUDA cache and retrying...")
+                        torch.cuda.empty_cache()
+                    else:
+                        print(f"Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                    continue
+                else:
+                    print(f"Runtime error: {e}")
+                    break
+            except Exception as e:
+                print(f"Error detecting text: {e}")
+                break
+        
+        print(f"Failed to detect text after {max_retries} attempts.")
+        return []
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the Text_Detector_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd octotools/tools/text_detector
+    python tool.py
+    """
+    import json
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Text_Detector_Tool
+    tool = Text_Detector_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    # relative_image_path = "examples/chinese_tra.jpg"
+    # relative_image_path = "examples/chinese.jpg"
+    relative_image_path = "examples/english.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+
+    # Execute the tool
+    try:
+        # execution = tool.execute(image=image_path, languages=["en", "ch_sim"])
+        # execution = tool.execute(image=image_path, languages=["en", "ch_tra"])
+        execution = tool.execute(image=image_path, languages=["en"])
+        print(json.dumps(execution))
+
+        print("Detected Text:", execution)
+    except ValueError as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/url_text_extractor/tool.py

@@ -0,0 +1,105 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+
+from octotools.tools.base import BaseTool
+
+class URL_Text_Extractor_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="URL_Text_Extractor_Tool",
+            tool_description="A tool that extracts all text from a given URL.",
+            tool_version="1.0.0",
+            input_types={
+                "url": "str - The URL from which to extract text.",
+            },
+            output_type="dict - A dictionary containing the extracted text and any error messages.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(url="https://example.com")',
+                    "description": "Extract all text from the example.com website."
+                },
+                {
+                    "command": 'execution = tool.execute(url="https://en.wikipedia.org/wiki/Python_(programming_language)")',
+                    "description": "Extract all text from the Wikipedia page about Python programming language."
+                },
+            ],
+        )
+
+    def extract_text_from_url(self, url):
+        """
+        Extracts all text from the given URL.
+
+        Parameters:
+            url (str): The URL from which to extract text.
+
+        Returns:
+            str: The extracted text.
+        """
+        url = url.replace("arxiv.org/pdf", "arxiv.org/abs")
+
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.content, 'html.parser')
+            text = soup.get_text(separator='\n', strip=True)
+            text = text[:10000] # Limit the text to 10000 characters
+            return text
+        except requests.RequestException as e:
+            return f"Error fetching URL: {str(e)}"
+        except Exception as e:
+            return f"Error extracting text: {str(e)}"
+
+    def execute(self, url):
+        extracted_text = self.extract_text_from_url(url)
+        return {
+            "url": url,
+            "extracted_text": extracted_text
+        }
+    
+    def get_metadata(self):
+        """
+        Returns the metadata for the URL_Text_Extractor_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+
+    cd octotools/tools/url_text_extractor
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the URL_Text_Extractor_Tool
+    tool = URL_Text_Extractor_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Sample URL for extracting text
+    url = "https://en.wikipedia.org/wiki/Python_(programming_language)"
+
+    import json
+
+    # Execute the tool with the sample URL
+    try:
+        execution = tool.execute(url=url)
+        print("Execution Result:")
+        print(json.dumps(execution, indent=4))
+        for key, value in execution.items():
+            print(f"{key}:\n{value}\n")
+    except ValueError as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")

octotools/tools/wikipedia_knowledge_searcher/tool.py

@@ -0,0 +1,130 @@
+import os
+import wikipedia
+
+from octotools.tools.base import BaseTool
+
+class Wikipedia_Knowledge_Searcher_Tool(BaseTool):
+    def __init__(self):
+        super().__init__(
+            tool_name="Wikipedia_Knowledge_Searcher_Tool",
+            tool_description="A tool that searches Wikipedia and returns web text based on a given query.",
+            tool_version="1.0.0",
+            input_types={
+                "query": "str - The search query for Wikipedia.",            },
+            output_type="dict - A dictionary containing the search results, extracted text, and any error messages.",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(query="Python programming language")',
+                    "description": "Search Wikipedia for information about Python programming language."
+                },
+                {
+                    "command": 'execution = tool.execute(query="Artificial Intelligence")',
+                    "description": "Search Wikipedia for information about Artificial Intelligence"
+                },
+                {
+                    "command": 'execution = tool.execute(query="Theory of Relativity")',
+                    "description": "Search Wikipedia for the full article about the Theory of Relativity."
+                },
+            ],
+        )
+
+    def search_wikipedia(self, query, max_length=2000):
+        """
+        Searches Wikipedia based on the given query and returns the text.
+
+        Parameters:
+            query (str): The search query for Wikipedia.
+            max_length (int): The maximum length of the returned text. Use -1 for full text.
+
+        Returns:
+            tuple: (search_results, page_text)
+        """
+        try:
+            search_results = wikipedia.search(query)
+            if not search_results:
+                return [], "No results found for the given query."
+
+            page = wikipedia.page(search_results[0])
+            text = page.content
+
+            if max_length != -1:
+                text = text[:max_length]
+
+            return search_results, text
+        except wikipedia.exceptions.DisambiguationError as e:
+            return e.options, f"DisambiguationError: {str(e)}"
+        except wikipedia.exceptions.PageError:
+            return [], f"PageError: No Wikipedia page found for '{query}'."
+        except Exception as e:
+            return [], f"Error searching Wikipedia: {str(e)}"
+
+    def execute(self, query, max_length=2000):
+        """
+        Searches Wikipedia based on the provided query and returns the results.
+
+        Parameters:
+            query (str): The search query for Wikipedia.
+            max_length (int): The maximum length of the returned text. Use -1 for full text.
+
+        Returns:
+            dict: A dictionary containing the search results, extracted text, and formatted output.
+        """
+        search_results, text = self.search_wikipedia(query, max_length)
+        
+        formatted_output = f"Search results for '{query}':\n"
+        formatted_output += "\n".join(f"{i}. {result}" for i, result in enumerate(search_results, 1))
+        formatted_output += f"\n\nExtracted text:\n{text}"
+
+        return {
+            # "search_results": search_results,
+            # "extracted_text": text,
+            "output": formatted_output
+        }
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the Wikipedia_Knowledge_Searcher_Tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = super().get_metadata()
+        return metadata
+
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+
+    cd octotools/tools/wikipedia_knowledge_searcher
+    python tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Example usage of the Wikipedia_Knowledge_Searcher_Tool
+    tool = Wikipedia_Knowledge_Searcher_Tool()
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Sample query for searching Wikipedia
+    # query = "Python programming language"
+    query = "kidney"
+
+    import json
+
+    # Execute the tool with the sample query
+    try:
+        execution = tool.execute(query=query)
+        print("Execution Result:")
+        print(json.dumps(execution, indent=4))
+        for key, value in execution.items():
+            print(f"{key}:\n{value}\n")
+    except ValueError as e:
+        print(f"Execution failed: {e}")
+
+    print("Done!")