Update variables.py

variables.py

@@ -1,6 +1,79 @@
 ##Variables
 
 import os
+import streamlit as st
+import pathlib
+
+from langchain.embeddings import HuggingFaceEmbeddings,HuggingFaceInstructEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.chat_models.openai import ChatOpenAI
+from langchain import VectorDBQA
+import pandas as pd
+
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    AIMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+)
+from langchain.schema import (
+    AIMessage,
+    HumanMessage,
+    SystemMessage
+)
+
+@st.experimental_singleton(suppress_st_warning=True)
+def get_latest_file():
+    '''Get the latest file from output folder'''
+
+    # set the directory path
+    directory_path = "output/"
+    
+    # create a list of all text files in the directory and sort by modification time
+    text_files = sorted(pathlib.Path(directory_path).glob("*.txt"), key=lambda f: f.stat().st_mtime)
+    
+    # get the latest modified file
+    latest_file = text_files[-1]
+    
+    # open the file and read its contents
+    with open(latest_file, "r") as f:
+        file_contents = f.read()
+
+    return file_contents
+
+@st.experimental_singleton(suppress_st_warning=True)
+def process_tweets(file,embed_model,query):
+    '''Process file with latest tweets'''
+
+    # Split tweets int chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    texts = text_splitter.split_text(file)
+
+    model = bi_enc_dict[embed_model]
+
+    if model == "hkunlp/instructor-large":
+        emb = HuggingFaceInstructEmbeddings(model_name=model,
+                                            query_instruction='Represent the Financial question for retrieving supporting documents: ',
+                                            embed_instruction='Represent the Financial document for retrieval: ')
+        
+    elif model == "sentence-transformers/all-mpnet-base-v2":
+        emb = HuggingFaceEmbeddings(model_name=model)
+
+    docsearch = FAISS.from_texts(texts, emb)
+
+    chain_type_kwargs = {"prompt": prompt}
+    chain = VectorDBQA.from_chain_type(
+    ChatOpenAI(temperature=0), 
+    chain_type="stuff", 
+    vectorstore=docsearch,
+    chain_type_kwargs=chain_type_kwargs
+    )
+
+    result = chain({"query": query}, return_only_outputs=True)
+
+    return result
 
 CONFIG = {
     "bearer_token": os.environ.get("bearer_token")