Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -3,6 +3,7 @@ from smolagents.tools import Tool
 import requests
 import markdownify
 import smolagents
+import re  # Add re import here
 
 class VisitWebpageTool(Tool):
     name = "visit_webpage"
@@ -22,9 +23,14 @@ class VisitWebpageTool(Tool):
                 "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
             ) from e
         try:
+            # Add user agent to avoid some blocking
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+            }
+            
             # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20)
-            response.raise_for_status()  # Raise an exception for bad status codes
+            response = requests.get(url, timeout=20, headers=headers)
+            response.raise_for_status()
 
             # Convert the HTML content to Markdown
             markdown_content = markdownify(response.text).strip()