Update utils.py

utils.py

@@ -1,69 +1,69 @@
-import os
-from langchain_groq import ChatGroq
-from langchain_community.vectorstores.faiss import FAISS
-from langchain_community.document_loaders import PyPDFDirectoryLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceHubEmbeddings
-from langchain.tools.retriever import create_retriever_tool
-from langchain_community.tools import ArxivQueryRun
-from langchain_community.utilities import ArxivAPIWrapper
-from langchain.agents import tool
-from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, PromptTemplate, MessagesPlaceholder
-from langchain import hub
-from googlesearch import search
-import requests
-from tqdm import tqdm
-from bs4 import BeautifulSoup
-
-def create_retriever_tool_agent(pdf_dir="./Pdfs"):
-    loader = PyPDFDirectoryLoader(pdf_dir)
-    documents = loader.load()
-    doc_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200
-    )
-    split_docs = doc_splitter.split_documents(documents)
-    embedding = HuggingFaceHubEmbeddings(model="sentence-transformers/all-MiniLM-L6-v2")
-    db = FAISS.from_documents(split_docs, embedding)
-    return create_retriever_tool(retriever=db.as_retriever(), name="Pdf_search", description="Use to search information from the PDFs. If information is not found, then use other tools.")
-
-def create_arxiv_tool_agent():
-    arxiv_api = ArxivAPIWrapper(top_k_results=1, doc_content_chars_max=200)
-    return ArxivQueryRun(api_wrapper=arxiv_api)
-
-@tool
-def google_search(input: str, num_results: int = 5):
-    """The Google Search tool enables the AI to fetch information from the web using Google's search engine. It is used when the AI's internal database does not have specific information requested by the user."""
-    search_results = [url for url in tqdm(search(input, num_results=num_results))]
-    return fetch_content(search_results)
-
-def fetch_content(urls: list):
-    text = []
-    for link in tqdm(urls):
-        try:
-            response = requests.get(link)
-            response.raise_for_status()
-            soup = BeautifulSoup(response.text, 'html.parser')
-            paragraphs = ' '.join([p.text for p in soup.find_all('p')])
-            paragraphs = paragraphs[:1000]
-            if paragraphs:
-                text.append(paragraphs)
-        except Exception:
-            continue
-    return " ".join(text)
-
-def get_prompt():
-    prompt = hub.pull("hwchase17/openai-functions-agent")
-    prompt.messages = [
-        SystemMessagePromptTemplate(prompt=PromptTemplate(
-            input_variables=[],
-            template="You are a helpful AI that answer from his database and if information is not found, then use a tool. You can use one tool at a time to search for information. Once a tool is used, do not switch to another tool."
-        )),
-        MessagesPlaceholder(variable_name='chat_history', optional=True),
-        HumanMessagePromptTemplate(prompt=PromptTemplate(
-            input_variables=['input'], 
-            template='{input}'
-        )),
-        MessagesPlaceholder(variable_name='agent_scratchpad')
-    ]
-    return prompt
+import os
+from langchain_groq import ChatGroq
+from langchain_community.vectorstores.faiss import FAISS
+from langchain_community.document_loaders import PyPDFDirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceHubEmbeddings
+from langchain.tools.retriever import create_retriever_tool
+from langchain_community.tools import ArxivQueryRun
+from langchain_community.utilities import ArxivAPIWrapper
+from langchain.agents import tool
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, PromptTemplate, MessagesPlaceholder
+from langchain import hub
+from googlesearch import search
+import requests
+from tqdm import tqdm
+from bs4 import BeautifulSoup
+
+def create_retriever_tool_agent(pdf_dir="./Pdfs"):
+    loader = PyPDFDirectoryLoader(pdf_dir)
+    documents = loader.load()
+    doc_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200
+    )
+    split_docs = doc_splitter.split_documents(documents)
+    embedding = HuggingFaceHubEmbeddings(model="sentence-transformers/all-MiniLM-L6-v2")
+    db = FAISS.from_documents(split_docs, embedding)
+    return create_retriever_tool(retriever=db.as_retriever(), name="Pdf_search", description="Use to search information from the PDFs first. If information is not found, then use other tools.")
+
+def create_arxiv_tool_agent():
+    arxiv_api = ArxivAPIWrapper(top_k_results=1, doc_content_chars_max=200)
+    return ArxivQueryRun(api_wrapper=arxiv_api)
+
+@tool
+def google_search(input: str, num_results: int = 5):
+    """The Google Search tool enables the AI to fetch information from the web using Google's search engine. It is used when the AI's internal database does not have specific information requested by the user."""
+    search_results = [url for url in tqdm(search(input, num_results=num_results))]
+    return fetch_content(search_results)
+
+def fetch_content(urls: list):
+    text = []
+    for link in tqdm(urls):
+        try:
+            response = requests.get(link)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, 'html.parser')
+            paragraphs = ' '.join([p.text for p in soup.find_all('p')])
+            paragraphs = paragraphs[:1000]
+            if paragraphs:
+                text.append(paragraphs)
+        except Exception:
+            continue
+    return " ".join(text)
+
+def get_prompt():
+    prompt = hub.pull("hwchase17/openai-functions-agent")
+    prompt.messages = [
+        SystemMessagePromptTemplate(prompt=PromptTemplate(
+            input_variables=[],
+            template="You are a helpful AI that uses tools to answer human questions. First try to answer from your database, and if information is not found, then use a tool. You can use one tool at a time to search for information. Once a tool is used, do not switch to another tool."
+        )),
+        MessagesPlaceholder(variable_name='chat_history', optional=True),
+        HumanMessagePromptTemplate(prompt=PromptTemplate(
+            input_variables=['input'], 
+            template='{input}'
+        )),
+        MessagesPlaceholder(variable_name='agent_scratchpad')
+    ]
+    return prompt