Update app.py

app.py

@@ -1,202 +1,66 @@
-import os
-import streamlit as st
-from dotenv import load_dotenv
-from PyPDF2 import PdfReader
-from langchain.text_splitter import CharacterTextSplitter
-from langchain.embeddings import HuggingFaceBgeEmbeddings
+from langchain.document_loaders import PyPDFDirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import FAISS
-from langchain.chat_models import ChatOpenAI
-from langchain.memory import ConversationBufferMemory
+from langchain.llms import openai
 from langchain.chains import ConversationalRetrievalChain
-from htmlTemplates import css, bot_template, user_template
-from langchain.llms import HuggingFaceHub
-from deep_translator import GoogleTranslator
-import pandas as pd
-from langchain_groq import ChatGroq
-from openai import OpenAI
 from langchain.chat_models import ChatOpenAI
-# set this key as an environment variable
-os.environ["HUGGINGFACEHUB_API_TOKEN"] = st.secrets['Key2']
-os.environ["OPENAI_API_KEY"] =st.secrets['Key3']
-from langchain.llms import LlamaCpp
-from langchain import PromptTemplate, LLMChain
-from langchain.callbacks.manager import CallbackManager
-from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-
+from langchain.embeddings import HuggingFaceBgeEmbeddings
 ###########################################################################################
 
-def get_pdf_text(pdf_docs : list) -> str:
-    text = ""
-    for pdf in pdf_docs:
-        pdf_reader = PdfReader(pdf)
-        for page in pdf_reader.pages:
-            text += page.extract_text()
-    return text
-#######################################################################################
-def load_file(): 
-    loader = TextLoader('d2.txt')
-    documents = loader.load()
-    return documents 
+def get_pdf_load():
+    loader=PyPDFDirectoryLoader("./data")
+    document=loader.load()
+    return document
 ########################################################################################
-def get_text_chunks(text:str) ->list:
-    text_splitter = CharacterTextSplitter(
-        separator="\n", chunk_size=1000, chunk_overlap=100, length_function=len
-    )
-    chunks = text_splitter.split_text(text)
-    return chunks
-
-
-def get_vectorstore(text_chunks : list) -> FAISS:
-    #model = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
-    model="Avditvs/multilingual-e5-small-distill-base-0.1"
-    encode_kwargs = {
-        "normalize_embeddings": True
-    }  # set True to compute cosine similarity
-    embeddings = HuggingFaceBgeEmbeddings(
-        model_name=model, encode_kwargs=encode_kwargs, model_kwargs={"device": "cpu"}
-    )
-    vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
-    return vectorstore
-
-
-def get_conversation_chain(vectorstore:FAISS) -> ConversationalRetrievalChain:
-    # llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613")
-    llm = HuggingFaceHub(
-        #repo_id="mistralai/Mistral-7B-Instruct-v0.2",
-        repo_id="google/gemma-1.1-7b-it",
-        #repo_id="TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF"
-        model_kwargs={"temperature": 0.5, "max_length": 2048},
-    )
-
-    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
-    conversation_chain = ConversationalRetrievalChain.from_llm(
-        llm=llm, retriever=vectorstore.as_retriever(), memory=memory
-    )
-    return conversation_chain
-
-
-
-def handle_userinput(user_question:str):
-    response = st.session_state.conversation({"question": user_question})
-    st.session_state.chat_history = response["chat_history"]
-
-    for i, message in enumerate(st.session_state.chat_history):
-        if i % 2 == 0:
-            text2=message.content 
-            translator = GoogleTranslator(source='english', target='persian')
-            result = translator.translate(text2)
-            st.write("سوال کاربر: "+result)
-        else:
-            text1=message.content 
-            translator = GoogleTranslator(source='english', target='persian')
-            result = translator.translate(text1)
-            st.write("پاسخ ربات: "+result)
-
-#############################################################################################################
-def read_pdf_pr_en(pdf_file_path):
-  from deep_translator import GoogleTranslator
-  import PyPDF2
-  # مسیر فایل PDF را تعیین کنید
-  #pdf_file_path = '/content/d2en.pdf'
-  # باز کردن فایل PDF
-  with open(pdf_file_path, 'rb') as pdf_file:
-    pdf_reader = PyPDF2.PdfReader(pdf_file)
-    # خواندن محتوای صفحه‌ها
-    full_text = ''
-    for page in pdf_reader.pages:
-        page_pdf=page.extract_text()
-        translator = GoogleTranslator(source='persian', target='english')
-        result = translator.translate(page_pdf)
-        full_text +=result
-    st.write(full_text)
-    return(full_text)
-#################################################################################################################
-def get_pdf_text(pdf_docs): 
-    text = "" 
-    for pdf in pdf_docs: 
-        pdf_reader = PdfReader(pdf) 
-    for page in pdf_reader.pages: 
-        txt_page=page.extract_text() 
-        text += txt_page
-    return text
-#######################################################################################################################
-def upload_xls():
-    st.title("آپلود و نمایش فایل اکسل")
-    uploaded_file = st.file_uploader("لطفاً فایل اکسل خود را آپلود کنید", type=["xlsx", "xls"])
-    if uploaded_file is not None:
-        df = pd.read_excel(uploaded_file)
-        st.write("دیتا فریم مربوط به فایل اکسل:")
-        st.write(df)
-    return df
-
-################################################################################################################
-def sentences_f(sentence,df2):
-  words = sentence.split()
-  df1 = pd.DataFrame(words, columns=['کلمات'])
-  df1['معادل'] = ''
-  for i, word in df1['کلمات'].items():
-    match = df2[df2['کلمات'] == word]
-    if not match.empty:
-        df1.at[i, 'معادل'] = match['معادل'].values[0]
-  df1['معادل'] = df1.apply(lambda row: row['کلمات'] if row['معادل'] == '' else row['معادل'], axis=1)
-  translated_sentence = ' '.join(df1['معادل'].tolist())
-  return translated_sentence
-####################################################################################################################    
-
+def get_text_split(document):
+    text_splitter= RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+    texts =text_splitter.split_documents(document)
+    return texts
+#########################################################################################
+def get_vectorstore(texts):
+    #Vector and Embeddings
+    DB_FAISS_PATH = 'vectore_Imstudio/faiss'
+    #Vector and Embeddings
+    embeddings= HuggingFaceBgeEmbeddings(model_name='Avditvs/multilingual-e5-small-distill-base-0.1', model_kwargs={'device': 'cpu'})
+    db= FAISS.from_documents(texts,embeddings)
+    db.save_local(DB_FAISS_PATH)
+    return db
+############################################################################################
+def get_chain(db):
+    llm=ChatOpenAI(base_url="https://bd4c-85-9-86-142.ngrok-free.app/v1", api_key="lm-studio",temperature=0.1,model="lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF")
+    #Build a chain
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        llm,db.as_retriever (search_kwargs={'k':2}),return_source_documents=True)
+    return  qa_chain
 ####################################################################################################################
 def main():
     st.set_page_config(
         page_title="Chat Bot PDFs",
         page_icon=":books:",
     )
-
-    #st.markdown("# Chat with a Bot")
-    #st.markdown("This bot tries to answer questions about multiple PDFs. Let the processing of the PDF finish before adding your question. 🙏🏾")
-
-    st.write(css, unsafe_allow_html=True)
-    #df2=upload_xls()
-
-    
-    if "conversation" not in st.session_state:
-        st.session_state.conversation = None
-    if "chat_history" not in st.session_state:
-        st.session_state.chat_history = None
-
     
     st.header("Chat Bot PDFs :books:")
     user_question = st.text_input("Ask a question about your documents:")
-    #user_question2=sentences_f(sentence=user_question1,df2=df2)
-    #translator = GoogleTranslator(source='persian', target='english')
-    #user_question = translator.translate(user_question2)
+    if st.button("Build Model"):
+        with st.spinner("Waiting"):
+            document=get_pdf_load()
+            texts=et_text_split(document)
+            db=get_vectorstore(texts)
+            qa_chain=get_chain(db)
+            st.write("compelete build model")
+  
     if st.button("Answer"):
             with st.spinner("Answering"):
-              handle_userinput(user_question)
+              
           
     if st.button("CLEAR"):
             with st.spinner("CLEARING"):
               st.cache_data.clear()
          
     
-    with st.sidebar:
-        st.subheader("Your documents")
-        pdf_docs = st.file_uploader("Upload your PDFs here and click on 'Process'", accept_multiple_files=True)
-
-        if st.button("Process"):
-            with st.spinner("Processing"):
-                # get pdf text
-                raw_text = get_pdf_text(pdf_docs)
-
-                # get the text chunks
-                text_chunks = get_text_chunks(raw_text)
-
-                # create vector store
-                vectorstore = get_vectorstore(text_chunks)
-
-                # create conversation chain
-                st.session_state.conversation = get_conversation_chain(vectorstore)
-                
-                #compelete build model
-                st.write("compelete build model")
+    #with st.sidebar:
+        #if st.button("Process build model"):
+ 
 
 
 if __name__ == "__main__":