adding new reddit group

app.py

@@ -1,188 +0,0 @@
-from langchain_core.messages import BaseMessage, HumanMessage
-from langchain_openai import ChatOpenAI
-from typing import Annotated
-import operator
-from typing import Sequence, TypedDict
-import numpy as np
-import pandas as pd
-from dotenv import load_dotenv
-import os
-from typing import Annotated
-import operator
-from typing import Sequence, TypedDict
-import matplotlib.pyplot as plt
-from langchain.schema.output_parser import StrOutputParser
-import streamlit as st
-import requests
-from requests import Request, Session
-from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
-import json
-
-st.set_page_config(page_title="LangChain Agent", layout="wide")
-load_dotenv()
-
-COINGECKO_API_KEY=os.environ["COINGECKO_API_KEY"]
-OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
-
-llm = ChatOpenAI(model="gpt-3.5-turbo")
-
-#======================== AGENTS ==================================
-# The agent state is the input to each node in the graph
-class AgentState(TypedDict):
-    # The annotation tells the graph that new messages will always
-    # be added to the current states
-    messages: Annotated[Sequence[BaseMessage], operator.add]
-    # The 'next' field indicates where to route to next
-    next: str
-             
-from langchain_core.runnables import RunnableConfig
-st.title("💬 Krypto")
-
-#@st.cache_data
-
-#@st.cache_resource
-#def initialize_session_state():
-if "chat_history" not in st.session_state:
-    st.session_state["messages"] = [{"role":"system", "content":"""
-How can I help you?
-"""}]
-
-#initialize_session_state()
-
-# Streamlit UI elements
-
-#st.text("Start by entering the currency.")
-
-sideb = st.sidebar
-
-with st.sidebar:
-    #st.subheader("This is the LangGraph workflow visualization of this application rendered in real-time.")
-    #st.image(create_graph_image())
-    
-    title = st.text_input("Start by entering the currency name:")
-
-check1 = sideb.button(f"analyze {title}")
-results=[]
-
-if check1:
-    st.write(f"I am now producing analysis for {title}")
-    
-    model = ChatOpenAI(temperature=0.7, api_key=OPENAI_API_KEY)
-    chain= model | StrOutputParser()
-    result=chain.invoke(f"You are a cryptocurrency data analyst.\
-                Provide correct cryptocurrency ticker from Coingecko website for cryptocurrency: {title}.\
-                Expected output: ticker.\
-                Provide it in the following format: >>cryptocurrencyticker>> \
-                for example: >>BTC>>")
-    
-
-    # for s in graph_data.stream(inputs):
-    #         for key, value in s.items():
-    #             print(f"Finished running: {value}:")
-    #             result = value["messages"][-1].content
-    #             results.append(value["messages"][-1].content)
-
-    print(result)
-    print('ticker',str(result).split(">>")[0])
-    if len(str(result).split(">>")[1])<10:
-        cryptocurrencyticker=(str(result).split(">>")[1])
-    else:
-        cryptocurrencyticker=(str(result).split(">>")[0])
-    cryptocurrency=title
-
-    print(cryptocurrency,cryptocurrencyticker)
-    print('here')
-
-    # # 1. Scrape historical Price and Volume  currency data
-    # from datetime import date
-    # today = date.today()
-    # Day_end = today.strftime("%d")
-    # Month_end = today.strftime("%m")
-    # Year_end = today.strftime("%Y")
-
-    # from datetime import date
-    # from datetime import timedelta
-
-    # past=today-timedelta(days=200)
-    # Day_start = past.strftime("%d")
-    # Month_start = past.strftime("%m")
-    # Year_start = past.strftime("%Y")
-
-    # date_start=[Year_start,Month_start,Day_start]
-    # date_end=[Year_end,Month_end,Day_end]
-
-    # import datetime
-    # import time
-
-    # #DATE definitions
-
-    # date_time = datetime.datetime(int(date_start[0]),int(date_start[1]),int(date_start[2]))
-    # date_time_now = datetime.datetime(int(date_end[0]),int(date_end[1]),int(date_end[2]))
-    # unix_past=time.mktime(date_time.timetuple()) #change the date format into unix for scraping
-    # unix_now=time.mktime(date_time_now.timetuple())
-    # past=datetime.datetime(int(date_start[0]),int(date_start[1]),int(date_start[2])).strftime('%Y-%m-%d')
-    # now=datetime.datetime(int(date_end[0]),int(date_end[1]),int(date_end[2])).strftime('%Y-%m-%d')
-    # datum_range=pd.date_range(start=past,end=now, freq='D')
-    
-    # #empty lists
-    # unix_all=[]
-    # coins_names=[]
-
-    # #create date variable
-    # for val in datum_range:
-    #     unix_all=np.append(unix_all,time.mktime(val.timetuple()))
-    # #from utils import slice
-    # # Get API for CoinGecko
-    # #cg = CoinGeckoAPI()
-
-    # url = f"https://api.coingecko.com/api/v3/coins/{cryptocurrency.lower()}/market_chart/range?vs_currency=usd&from={unix_past}&to={unix_now}"
-
-    # headers = {
-    #     "accept": "application/json",
-    #     "x-cg-demo-api-key": COINGECKO_API_KEY
-    # }
-    # response = ''
-    # while response == '':
-    #     try:
-    #         response = requests.get(url, headers=headers, proxies={"http": "http://111.233.225.166:1234"})
-    #         break
-    #     except:
-    #         print("Connection refused by the server..")
-    #         print("Let me sleep for 5 seconds")
-    #         print("ZZzzzz...")
-    #         time.sleep(5)
-    #         print("Was a nice sleep, now let me continue...")
-    #         continue
-    
-    # data=response.json()
-
-     #This example uses Python 2.7 and the python-request library.
-
-
-
-    url = 'https://sandbox-api.coinmarketcap.com/v1/cryptocurrency/listings/latest'
-    parameters = {
-    'start':'1',
-    'limit':'5000',
-    'convert':'USD'
-    }
-    headers = {
-    'Accepts': 'application/json',
-    'X-CMC_PRO_API_KEY': 'b54bcf4d-1bca-4e8e-9a24-22ff2c3d462c',
-    }
-
-    session = Session()
-    session.headers.update(headers)
-
-    try:
-        response = session.get(url, params=parameters)
-        data = json.loads(response.text)
-        print(data)
-    except (ConnectionError, Timeout, TooManyRedirects) as e:
-        print(e)
-
-    #data=cg.get_coin_market_chart_range_by_id(id=cryptocurrency.lower(),vs_currency='usd',include_market_cap='true', include_24hr_vol='true', from_timestamp=unix_past,to_timestamp=unix_now)
-    #df_ts_coins=su.scrape_historical_series([currency],date_start,date_end)
-    #================== Scrape Current/Historical Price ====================
-    st.write(data)
-        

app_crypto_arima_model.py

@@ -0,0 +1,76 @@
+from datetime import datetime, timedelta
+import pandas as pd
+import numpy as np
+
+import model_utils as mu
+from statsmodels.tsa.arima.model import ARIMA
+
+def model_run(df_all):
+    """ Prediciton function that runs ARIMA model and predicts tomorrow cryptocurrency price.
+    Useful for forecasting a variable using ARIMA model.
+    Use historical 'prices' and get prediction.
+    Give prediction output to the client.
+    """
+    first_day_future=pd.to_datetime(datetime.now()+timedelta(days=1))
+    #----------------------------------------- DATASET MANIPULATION FOR SUPERVISED LEARNING --------------------------------------------
+    reframed_lags, df_final=mu.data_transform(df_all, first_day_future)
+    
+    print(f'I have transformed the dataset into the frame for supervised learning')
+    df=reframed_lags[['prices','price_eth','GSPC','Day','Month', 'TNX', 'Employment', 'google_trend','EURUSD']]
+    date=pd.to_datetime(dict(year=reframed_lags['Year'], month=reframed_lags['Month'], day=reframed_lags['Day']))
+    df_with_date=pd.concat([date,df],axis=1)
+    df_with_date.columns=np.append('date',df.columns)
+    df_with_date.set_index('date',inplace=True)
+    df_with_date=df_with_date.dropna()
+    df_past=df_with_date.iloc[:-1,:]
+    df_future=df_with_date.iloc[-1:,:]
+    model = ARIMA(df_past['prices'],exog=df_past.drop(columns=['prices']), order=(2,1,2))
+    model_fit = model.fit()
+
+    # Make predictions
+    predictions = model_fit.forecast(steps=1,exog=df_future.drop(columns='prices'))
+
+    #Add forecast to df_with_date
+    df_with_forecast=reframed_lags.copy()
+    df_with_forecast.loc[df_with_forecast.index==df_with_forecast.index[-1],'prices']=predictions[-1:].values[0]
+    #----------------------------------- MODEL ACCURACY
+    #Calculate accuracy after transformation!!!
+    #get rid of values below 0.01 which skew the accuracy measure if in denominator
+
+    #Rolling window accuracy measure
+    if len(reframed_lags)>500:
+        train_size=0.9
+    elif len(reframed_lags)>200:
+        train_size=0.8
+    else:
+        train_size=0.7
+    predictions=[]
+    test_labels_all=[]
+    test_labels_all1=[]
+    train_labels_all=[]
+    data_arima=df_with_date
+    window_length=int((len(data_arima)-len(data_arima)*train_size))
+    for i in range(0,window_length):
+        train_accuracy=data_arima.iloc[0:int(len(data_arima)*train_size)+i,:]
+        
+        test_accuracy=data_arima.iloc[len(train_accuracy):len(train_accuracy)+1,:]
+        train_features_accuracy=train_accuracy.drop(columns='prices')
+        test_features_accuracy=test_accuracy.drop(columns='prices')
+        train_labels_accuracy=train_accuracy['prices']
+        test_labels_accuracy=test_accuracy['prices']
+        print(train_labels_accuracy)
+
+        arima = ARIMA(train_labels_accuracy,exog=train_features_accuracy, order=(2,1,2)) #RandomForestRegressor(n_estimators= 1000)
+        arima_fit=arima.fit() #train_features_accuracy, train_labels_accuracy)
+        prediction_arima = arima_fit.forecast(steps=1,exog=test_features_accuracy) #predict(test_features_accuracy)
+        predictions=np.append(predictions,prediction_arima)
+        test_labels_all=np.append(test_labels_all,test_labels_accuracy)
+        train_labels_all=np.append(train_labels_all,train_accuracy)
+        test_labels_all1=np.append(test_labels_all1,test_accuracy)
+
+    #Calculate accuracy
+    from sklearn.metrics import r2_score
+    accuracy=r2_score(predictions,test_labels_all)
+    result_arima=pd.DataFrame({'prediction':predictions,'data':test_labels_all})
+    result_arima.to_csv('result_arima_kat.csv')
+    return df_with_forecast, accuracy, result_arima

app_crypto_rf_model.py

@@ -0,0 +1,124 @@
+from datetime import datetime, timedelta
+import pandas as pd
+import numpy as np
+
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_error
+from math import sqrt
+from sklearn.preprocessing import MinMaxScaler
+import model_utils as mu
+
+def model_run(df_all):
+    """ Prediciton function that runs random forest model and predicts tomorrow cryptocurrency price"""
+    
+    first_day_future=pd.to_datetime(datetime.now()+timedelta(days=1))
+    #----------------------------------------- DATASET MANIPULATION FOR SUPERVISED LEARNING --------------------------------------------
+    reframed_lags, df_final=mu.data_transform(df_all, first_day_future)
+    print(f'I have transformed the dataset into the frame for supervised learning')
+    reframed_lags.to_csv('reframed_lags.csv')
+    #----------------------------------------- TRAIN/TEST SPLIT ------------------------------------------------------
+    """ Randomly split a chunk into train test based on train/test ratio (0.8) and split the other chunks for all the other currencies in the same fashion"""
+    import random
+    train_size=0.8
+
+    df_cut1=reframed_lags.reset_index().iloc[:,1:]
+    print('tady')
+    train_value=int(len(df_cut1)*train_size)
+    first_random=random.sample(range(len(df_cut1)-1), train_value)
+    train_bulk=np.sort(first_random) #make sure all the consequent ones have the same random numbers
+
+    df_cut=reframed_lags.reset_index()
+    train_sample=df_cut.loc[df_cut['index'].isin(train_bulk)]
+    test_sample=df_cut.loc[~df_cut['index'].isin(train_bulk)]
+
+    test=test_sample.iloc[:,1:] 
+    train=train_sample.iloc[:,1:]
+    print(f'I have split the dataset into training and testing samples')
+
+    #----------------------------------- Re-Scale for supervised learning
+    # TRAIN RESCALE
+    # normalize features for the supervised learning (0,1)
+    scaler_train = MinMaxScaler(feature_range=(0, 1))
+    scaled = scaler_train.fit_transform(train.values.astype('float32'))
+    df_train=pd.DataFrame(scaled)
+    df_train.columns=train.columns #rename columns
+
+    # TEST RESCALE
+    scaler_test = MinMaxScaler(feature_range=(0, 1))
+    scaled = scaler_test.fit_transform(test.values.astype('float32'))
+    df_test=pd.DataFrame(scaled)
+    df_test.columns=test.columns #rename columns
+
+    #----------------------------------- MODEL
+
+    #define features
+    train_features=df_train.values
+    test_features=df_test.values
+    #define labels
+    train_labels = df_train['prices'].values
+    test_labels = df_test['prices'].values        
+
+    #define baseline prediction (as last values) for evaluating prediction accuracy
+    baseline_preds = pd.DataFrame(test_features).iloc[:,0]
+    # Calculate errors for the baseline prediction
+    baseline_errors = abs(baseline_preds - test_labels)
+
+    # Import the model we are using
+    from sklearn.ensemble import RandomForestRegressor
+    # Instantiate model with 1000 decision trees
+    rf = RandomForestRegressor(n_estimators= 1000)
+    rf.fit(train_features, train_labels)
+    prediction_rf = rf.predict(test_features)
+    predictions=prediction_rf
+
+    #----------------------------------- MODEL OUTPUT TRANSFORMATION
+    #Convert test column
+    df_test['prices']=predictions
+    prediction_transformed=pd.DataFrame(scaler_test.inverse_transform(df_test.values.astype('float')))
+    prediction_transformed.columns=test.columns
+
+    #Convert prediction
+    df_test.loc[df_test.index==(len(df_test)-1),'prices']=predictions[-1:][0]
+    inv_transformed=pd.DataFrame(scaler_test.inverse_transform(df_test.values.astype('float')))
+    inv_transformed.columns=test.columns
+
+    # data with forecast
+    df_with_forecast=df_final.copy()
+    df_with_forecast.loc[df_with_forecast.index==df_with_forecast.index[-1],'prices']=inv_transformed['prices'][-1:].values[0]
+    print('Final result')
+    print(df_with_forecast)
+
+    #----------------------------------- MODEL ACCURACY
+    #Calculate accuracy after transformation!!!
+    #get rid of values below 0.01 which skew the accuracy measure if in denominator
+
+    #Rolling window accuracy measure
+    if len(reframed_lags)>500:
+        train_size=0.9
+    elif len(reframed_lags)>200:
+        train_size=0.8
+    else:
+        train_size=0.7
+    predictions=[]
+    test_labels_all=[]
+    window_length=int((len(reframed_lags)-len(reframed_lags)*train_size))
+    for i in range(0,window_length):
+        train_accuracy=reframed_lags.iloc[0:int(len(reframed_lags)*train_size)+i,:]
+        test_accuracy=reframed_lags.iloc[len(train_accuracy):len(train_accuracy)+1,:]
+        train_features_accuracy=train_accuracy.drop(columns='prices')
+        test_features_accuracy=test_accuracy.drop(columns='prices')
+        train_labels_accuracy=train_accuracy['prices']
+        test_labels_accuracy=test_accuracy['prices']
+
+        rf = RandomForestRegressor(n_estimators= 1000)
+        rf.fit(train_features_accuracy, train_labels_accuracy)
+        prediction_rf = rf.predict(test_features_accuracy)
+        predictions=np.append(predictions,prediction_rf)
+        test_labels_all=np.append(test_labels_all,test_labels_accuracy)
+
+    #Calculate accuracy
+    from sklearn.metrics import r2_score
+    accuracy=r2_score(predictions,test_labels_all)
+    result_rf=pd.DataFrame({'prediction':predictions,'data':test_labels_all})
+    result_rf.to_csv('result_rf.csv')
+    return df_with_forecast, accuracy, result_rf

app_crypto_scrape.py

@@ -0,0 +1,82 @@
+
+#Import packages
+import pandas as pd
+import numpy as np
+from pycoingecko import CoinGeckoAPI
+cg = CoinGeckoAPI()
+import pandas as pd
+import numpy as np
+from pytrends.request import TrendReq
+pytrends = TrendReq(hl='en-US')
+import scrape_utils as su
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+COINMARKET_API_KEY=os.environ["COINMARKET_API_KEY"]
+
+def scrape_crypto(currency, ticker):
+
+    # 1. Scrape historical Price and Volume  currency data
+    from datetime import date
+    today = date.today()
+    Day_end = today.strftime("%d")
+    Month_end = today.strftime("%m")
+    Year_end = today.strftime("%Y")
+
+    from datetime import date
+    from datetime import timedelta
+
+    past=today-timedelta(days=300)
+    Day_start = past.strftime("%d")
+    Month_start = past.strftime("%m")
+    Year_start = past.strftime("%Y")
+
+    date_start=[Year_start,Month_start,Day_start]
+    date_end=[Year_end,Month_end,Day_end]
+
+    import datetime
+    df_ts_coins=su.scrape_historical_series([currency],ticker,date_start,date_end)[0]
+    print(df_ts_coins)
+    df_today_row=su.scrape_historical_series([currency],ticker,date_start,date_end)[1]
+    #print(df_today_row)
+
+    if len(df_ts_coins)>0:
+        print(df_today_row)
+        #df_today_row=df_today_row.drop(0)
+
+        df_ts_coins=df_ts_coins[['id','date','prices','market_caps','total_vol']]
+        df_ts_coins=pd.concat([df_ts_coins,df_today_row],axis=0)
+        df_ts_coins.set_index('date',inplace=True)
+        df_ts_coins.index=[pd.to_datetime(df_ts_coins.index[i]).strftime("%Y-%m-%d %H:%M:%S") for i in range(len(df_ts_coins))]
+
+        # 2. Scrape macro
+        df_cli=su.scrape_cli(past,today)
+        df_cpi=su.scrape_cpi_employment()
+        print(f'I have scraped CLI and L, CPI')
+
+        # 3. Scrape google-trends
+        google_data=su.scrape_google_trends(currency,ticker)
+        print(f'Google trend dataset')
+
+        # 4. Scrape Yahoo-Finance
+        df_finance=su.scrape_stocks(past,today)
+        print(f'yahoo dataset. I am done scraping !!!!!!!')
+
+        #==== 5. CONCAT DATAFRAMES TOGETHER
+        df_ts_coins.index=pd.to_datetime(df_ts_coins.index).strftime("%Y-%m-%d")
+        df_cli.index=pd.to_datetime(df_cli.index).strftime("%Y-%m-%d")
+        if len(df_cpi)>0:
+            df_cpi.index=pd.to_datetime(df_cpi.index).strftime("%Y-%m-%d")
+        else:
+            print('MISSING CPI')
+            df_cpi=pd.DataFrame({'CPI':np.repeat(0,len(df_cli)),'Employment':np.repeat(0,len(df_cli))})
+            df_cpi.index=df_cli.index
+        google_data.index=pd.to_datetime(google_data.index).strftime("%Y-%m-%d")
+        df_finance.index=pd.to_datetime(df_finance.index).strftime("%Y-%m-%d")
+        df_all=pd.concat([df_ts_coins,df_cli,df_cpi,google_data,df_finance],axis=1)
+        df_all=df_all.sort_index()
+    else:
+        print('No data available.')
+        df_all=pd.DataFrame()
+    return df_all

app_hf.py

@@ -0,0 +1,225 @@
+from openai import OpenAI
+from langchain.agents import AgentExecutor, create_openai_tools_agent
+from langchain_core.messages import BaseMessage, HumanMessage
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from typing import Annotated
+import operator
+from typing import Sequence, TypedDict
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+import numpy as np
+import pandas as pd
+from dotenv import load_dotenv
+import os
+from typing import Annotated
+import operator
+from typing import Sequence, TypedDict
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+import matplotlib.pyplot as plt
+from langchain.schema.output_parser import StrOutputParser
+from tools import data_analyst #forecasting_expert_arima, forecasting_expert_rf, evaluator, investment_advisor
+from tools import crypto_sentiment_analysis_util
+import app_crypto_rf_model as rf
+import app_crypto_scrape as sa
+import app_crypto_arima_model as arima
+import streamlit as st
+
+from datetime import date
+today = date.today()
+
+st.set_page_config(page_title="LangChain Agent", layout="wide")
+load_dotenv()
+OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+
+llm = ChatOpenAI(model="gpt-3.5-turbo")
+
+#======================== AGENTS ==================================
+# The agent state is the input to each node in the graph
+class AgentState(TypedDict):
+    # The annotation tells the graph that new messages will always
+    # be added to the current states
+    messages: Annotated[Sequence[BaseMessage], operator.add]
+    # The 'next' field indicates where to route to next
+    next: str
+             
+tool=data_analyst.data_analyst_tools()
+
+from langchain_core.runnables import RunnableConfig
+st.title("💬 Krypto")
+
+#@st.cache_data
+
+#@st.cache_resource
+#def initialize_session_state():
+if "chat_history" not in st.session_state:
+    st.session_state["messages"] = [{"role":"system", "content":"""
+How can I help you?
+"""}]
+
+#initialize_session_state()
+
+# Streamlit UI elements
+st.image('crypto_image.png')
+#st.text("Start by entering the currency.")
+
+sideb = st.sidebar
+
+with st.sidebar:
+    #st.subheader("This is the LangGraph workflow visualization of this application rendered in real-time.")
+    #st.image(create_graph_image())
+    
+    title = st.text_input("Start by entering the currency name:")
+
+check1 = sideb.button(f"analyze {title}")
+results=[]
+
+if check1:
+    st.write(f"I am now producing analysis for {title}")
+    
+    model = ChatOpenAI(temperature=0.7, api_key=OPENAI_API_KEY)
+    chain= model | StrOutputParser()
+    result=chain.invoke(f"You are a cryptocurrency data analyst.\
+                Provide correct cryptocurrency ticker from Coingecko website for cryptocurrency: {title}.\
+                Expected output: ticker.\
+                Provide it in the following format: >>cryptocurrencyticker>> \
+                for example: >>BTC>>")
+    
+
+    # for s in graph_data.stream(inputs):
+    #         for key, value in s.items():
+    #             print(f"Finished running: {value}:")
+    #             result = value["messages"][-1].content
+    #             results.append(value["messages"][-1].content)
+
+    print(result)
+    print('ticker',str(result).split(">>")[0])
+    if len(str(result).split(">>")[1])<10:
+        cryptocurrencyticker=(str(result).split(">>")[1])
+    else:
+        cryptocurrencyticker=(str(result).split(">>")[0])
+    cryptocurrency=title
+
+    print(cryptocurrency,cryptocurrencyticker)
+    print('here')
+
+    #================== Scrape Current/Historical Price ====================
+    df=sa.scrape_crypto(cryptocurrency,cryptocurrencyticker)
+    if len(df)>0:
+        print("Running forecasting models on historical prices")
+        df_with_forecast_rf, accuracy_rf, result_rf=rf.model_run(df)
+        
+        df_with_forecast_arima, accuracy_arima, result_arima=arima.model_run(df)
+
+        #--- for llm
+        if accuracy_rf<accuracy_arima:
+            forecasted_price=(np.round(np.array(df_with_forecast_arima['prices'])[-1]),2)
+            prompt = f"You are an investment recommendation expert for crypto currency {cryptocurrency}.You are selecting the predicted price from the ARIMA model because its accuracy (R2 measure:{(np.round(accuracy_arima,2))}) is higher than the accuracy (R2:{(np.round(accuracy_rf,2))}) for random forest model.Compare current price to the predicted price. If current price exceeds predicted price, recommend selling the stock, otherwise recommend buying. Tell the user what the current price, predicted price and accuracy values are. You know that the predicted price for tomorrow using random forest model is {(np.round(np.array(df_with_forecast_rf['prices'])[-1],2))}. The prediction accuracy for the random forest model is {(np.round(accuracy_rf,2))}. The current price of {cryptocurrency} is: {(np.round(df['prices'][-1],2))}. "
+        
+
+        else:
+            forecasted_price=(np.round(np.array(df_with_forecast_rf['prices'])[-1],2))
+            prompt = f"You are an investment recommendation expert for crypto currency {cryptocurrency}. You are selecting the predicted price from the random forest model because its accuracy (R2 measure:{(np.round(accuracy_rf,2))}) is higher than the accuracy (R2:{(np.round(accuracy_arima,2))}) for arima model. Compare current price to the predicted price. If current price exceeds predicted price, recommend selling the stock, otherwise recommend buying. Tell the user what the current price, predicted price and accuracy values are. You know that the predicted price for tomorrow using random forest model is {(np.round(np.array(df_with_forecast_arima['prices'])[-1]),2)}. The prediction accuracy for the random forest model is {(np.round(accuracy_arima,2))}. The current price of {cryptocurrency} is: {(np.round(df['prices'][-1],2))}. "
+        current_forecast=pd.read_csv('current_forecast.csv',index_col='date',parse_dates=True,infer_datetime_format=True)        
+        today=pd.to_datetime(today).strftime('%Y-%m-%d')
+        print([(np.array(df_with_forecast_arima['prices'])[-1]),np.array(df_with_forecast_rf['prices'])[-1],today])
+
+        if today not in (current_forecast.index):
+            prices_arima=np.append(current_forecast['prices_arima'],(np.array(df_with_forecast_arima['prices'])[-1]))
+            prices_rf=np.append(current_forecast['prices_rf'],(np.array(df_with_forecast_rf['prices'])[-1]))
+            dates=np.append(current_forecast.index[0].strftime('%Y-%m-%d'),today)
+            current_forecast=pd.DataFrame({'date':dates, 'prices_rf':prices_rf,'prices_arima':prices_arima})
+            current_forecast.to_csv('current_forecast.csv')
+
+        #prompt=str(prompt)
+        inputs_reccommend = {"messages": [HumanMessage(content=prompt)]}
+
+        model = ChatOpenAI(temperature=0.7, api_key=OPENAI_API_KEY)
+        response=model.invoke(prompt)
+        response_content=response.content
+        st.chat_message("assistant").markdown((response_content))
+        st.session_state.messages.append({"role": "assistant", "content": prompt})
+
+        fig, ax = plt.subplots(1,2, figsize=(10, 3))
+        ax[0].plot(result_arima['prediction'], color='blue', marker='o')
+        ax[0].plot(result_arima['data'], color='orange', marker='o')
+        ax[0].set_title('ARIMA')
+        ax[1].plot(result_rf['prediction'], color='blue', marker='o')
+        ax[1].plot(result_rf['data'], color='orange', marker='o')
+        ax[1].set_title('RF')
+        fig.suptitle('Prediction vs Actuals')
+        plt.legend(['prediction','actuals'])
+        st.pyplot(fig)
+    # ========================== Sentiment analysis
+    #Perform sentiment analysis on the cryptocurrency news & predict dominant sentiment along with plotting the sentiment breakdown chart
+    # Downloading from reddit
+    
+    # Downloading from alpaca
+    news_articles = crypto_sentiment_analysis_util.fetch_news(cryptocurrency)
+    reddit_news_articles=crypto_sentiment_analysis_util.fetch_reddit_news(cryptocurrency)
+    #os.system('scrapy crawl reddit -o crypto_reddit.txt')
+
+
+    #crypto_sentiment_analysis_util.fetch_reddit_news() #(f"cryptocurrency {cryptocurrency}")
+    analysis_results = []
+    
+    #Perform sentiment analysis for each product review
+    for article in news_articles:
+        if cryptocurrency[0:6] in article['News_Article'].lower():
+            sentiment_analysis_result = crypto_sentiment_analysis_util.analyze_sentiment(article['News_Article'])
+
+            # Display sentiment analysis results
+            #print(f'News Article: {sentiment_analysis_result["News_Article"]} : Sentiment: {sentiment_analysis_result["Sentiment"]}', '\n')
+
+            result = {
+                        'News_Article': sentiment_analysis_result["News_Article"],
+                        'Sentiment': sentiment_analysis_result["Sentiment"][0]['label'],
+                        'Index': sentiment_analysis_result["Sentiment"][0]['score']
+                    }
+            
+            analysis_results.append(result)
+
+    for article in reddit_news_articles:
+        if cryptocurrency[0:6] in article.lower():
+            sentiment_analysis_result_reddit = crypto_sentiment_analysis_util.analyze_sentiment(article)
+
+            # Display sentiment analysis results
+            #print(f'News Article: {sentiment_analysis_result_reddit["News_Article"]} : Sentiment: {sentiment_analysis_result_reddit["Sentiment"]}', '\n')
+
+            result = {
+                        'News_Article': sentiment_analysis_result_reddit["News_Article"],
+                        'Index':np.round(sentiment_analysis_result_reddit["Sentiment"][0]['score'],2)
+                    }
+            analysis_results.append(result)
+
+    #Generate summarized message rationalize dominant sentiment
+    summary = crypto_sentiment_analysis_util.generate_summary_of_sentiment(analysis_results)
+    st.chat_message("assistant").write(str(summary))
+    st.session_state.messages.append({"role": "assistant", "content": summary})
+    #answers=np.append(res["messages"][-1].content,summary)
+
+# Set OpenAI API key from Streamlit secrets
+client = OpenAI(api_key=OPENAI_API_KEY)
+
+# Set a default model
+if "openai_model" not in st.session_state:
+    st.session_state["openai_model"] = "gpt-3.5-turbo"
+
+#model = ChatOpenAI(temperature=0.7, api_key=OPENAI_API_KEY)
+if prompt := st.chat_input("Some other questions?"):
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+    # Display assistant response in chat message container
+    with st.chat_message("assistant"):
+        stream = client.chat.completions.create(
+            model=st.session_state["openai_model"],
+            messages=[
+                {"role": m["role"], "content": m["content"]}
+                for m in st.session_state.messages
+            ],
+            stream=True,
+        )
+        response = st.write_stream(stream)
+    st.session_state.messages.append({"role": "assistant", "content": response})

requirements.txt

@@ -1,12 +1,11 @@
-alpaca_trade_api
-transformers
-bitsandbytes
-yfinance
-gradio==4.42.0 
 tf-keras==2.17.0
 python-dotenv==1.0.1
+
 beautifulsoup4==4.12.3
+
+
 fastapi==0.110.3
+
 GoogleNews==1.6.15
 
 langchain==0.2.14
@@ -17,18 +16,8 @@ langchain-openai==0.1.21
 
 openai==1.40.8
 transformers==4.44.0
+
 pandas==2.2.2
-numpy==1.26.4
 praw==7.7.1
 streamlit==1.37.1
-typing-inspect==0.9.0
 
-matplotlib==3.9.2
-statsmodels==0.14.2
-scikit-learn==1.5.1
-pycoingecko==3.1.0
-beautifulsoup4==4.12.3
-requests==2.32.3
-pytrends==4.9.2
-yfinance==0.2.41
-prettytable==3.11.0

scrape_utils.py

@@ -19,11 +19,14 @@ import yfinance as yf
 import json
 import prettytable
 import os
+from requests import Request, Session
+from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
+import json
 load_dotenv()
 
-COINGECKO_API_KEY=os.environ["COINGECKO_API_KEY"]
+COINMARKET_API_KEY=os.environ["COINMARKET_API_KEY"]
 # Historical crypto data
-def scrape_historical_series(coin_name,date_start,date_end):
+def scrape_historical_series(coin_name,symbol,date_start,date_end):
     import datetime
     """ Scrape historical series on the sample of coins.
 
@@ -49,65 +52,143 @@ def scrape_historical_series(coin_name,date_start,date_end):
     
     #empty lists
     unix_all=[]
-    coins_names=[]
 
     #create date variable
     for val in datum_range:
         unix_all=np.append(unix_all,time.mktime(val.timetuple()))
-    
-    for coin in pd.unique(coin_name):
-        time.sleep(5)
-        url = f"https://api.coingecko.com/api/v3/coins/{coin.lower()}/market_chart/range?vs_currency=usd&from={unix_past}&to={unix_now}"
-
-        headers = {
-            "accept": "application/json",
-            "x-cg-demo-api-key": COINGECKO_API_KEY
-        }
-
-        response = requests.get(url, headers=headers)
-        data=response.json()
-        #data=cg.get_coin_market_chart_range_by_id(id=coin.lower(),vs_currency='usd',include_market_cap='true', include_24hr_vol='true', from_timestamp=unix_past,to_timestamp=unix_now)
-        if len(data)>0:
-            prices=pd.DataFrame(data['prices'],columns=['date','prices'])
-            market=pd.DataFrame(data['market_caps'],columns=['date','market_caps'])
-            volume=pd.DataFrame(data['total_volumes'],columns=['date','total_vol'])
-            ts_coins_cut=pd.concat([prices,market.iloc[:,1],volume.iloc[:,1]],axis=1)
-
-            #create id variable for each coin
-            coinn=np.repeat(coin,len(ts_coins_cut)) 
-            coins_names=np.append(coins_names,coinn)
-            
-            #make daily data from hourly
-            ts_coins_cut['id']=coinn
-            date_all=[]
-
-            #create date variable
-            import datetime
-            for val in ts_coins_cut['date']:
-                date_all=np.append(date_all,((datetime.datetime.fromtimestamp(int(val)/1000)).strftime('%m/%d/%y, %H:%M:%S')))
-            dates=pd.to_datetime(date_all, format='%m/%d/%y, %H:%M:%S')
-            
-            #set date as an index to aggreggate hourly data into daily
-            ts_coins_cut['dates']=dates
-            ts_coins_cut=ts_coins_cut.set_index('dates')
-            prices=ts_coins_cut.pop('prices')
-            ts_coins_cut=ts_coins_cut.groupby([pd.Grouper(freq='D'), 'id']).mean()
-            prices1=prices.groupby([pd.Grouper(freq='D')]).mean()
-            #after you aggreggated data change the index back
-            prices1=prices1.reset_index()
-            ts_coins_cut.reset_index(inplace=True)
-
-            ts_coins_cut.insert(2,'prices',prices1.iloc[:,1])
-            #move the date column to different position
-            ts_coins_cut=ts_coins_cut.drop(columns=['date']) 
-            ts_coins_cut.insert(2,'date',unix_all[0:len(ts_coins_cut)])
-            df_ts_coins1=pd.concat([df_ts_coins1,ts_coins_cut]) #concat the chunk with the selected variables across all currencies
-
-        
-        else:
-            df_ts_coins1=pd.DataFrame()
-        df_ts_coins1=df_ts_coins1.drop(columns=['dates'])
-    return df_ts_coins1
+    url = ' https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/historical'
+    parameters = {
+    'time_start': (int(unix_past)),
+    'time_end': (int(unix_now)),
+    'symbol': symbol,
+    'convert':'USD',
+    'interval': 'daily',
+    }
+    headers = {
+    'Accepts': 'application/json',
+    'X-CMC_PRO_API_KEY': COINMARKET_API_KEY,
+    }
+
+    session = Session()
+    session.headers.update(headers)
+
+    try:
+        response = session.get(url, params=parameters)
+        data_json = json.loads(response.text)
+        #data = json.loads(response.text['data']['quote']['USD'])
+
+    except (ConnectionError, Timeout, TooManyRedirects) as e:
+        print(e)
+
+    #SCRAPE FOR ETH
+#create date variable
+    for val in datum_range:
+        unix_all=np.append(unix_all,time.mktime(val.timetuple()))
+    url = ' https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/historical'
+    parameters = {
+    'time_start': (int(unix_past)),
+    'time_end': (int(unix_now)),
+    'symbol': 'ETH',
+    'convert':'USD',
+    'interval': 'daily',
+    }
+    headers = {
+    'Accepts': 'application/json',
+    'X-CMC_PRO_API_KEY': COINMARKET_API_KEY,
+    }
+
+    session = Session()
+    session.headers.update(headers)
+
+    try:
+        response = session.get(url, params=parameters)
+        data_json_eth = json.loads(response.text)
+        #data = json.loads(response.text['data']['quote']['USD'])
+
+    except (ConnectionError, Timeout, TooManyRedirects) as e:
+        print(e)
+
+    date=[]
+    price=[]
+    price_eth=[]
+    market_caps=[]
+    total_volumes=[]
+
+    for i in range(len(data_json['data'][symbol][0]['quotes'])):
+        date=np.append(date,data_json['data'][symbol][0]['quotes'][i]['quote']['USD']['timestamp'])
+        price=np.append(price,data_json['data'][symbol][0]['quotes'][i]['quote']['USD']['price'])
+        market_caps=np.append(market_caps,data_json['data'][symbol][0]['quotes'][i]['quote']['USD']['market_cap'])
+        total_volumes=np.append(total_volumes,data_json['data'][symbol][0]['quotes'][i]['quote']['USD']['volume_24h'])
+        price_eth=np.append(price_eth,data_json_eth['data']['ETH'][0]['quotes'][i]['quote']['USD']['price'])
+    ts_coins_cut=pd.DataFrame({'date':date, 'prices':price,'market_caps':market_caps,'total_vol':total_volumes,'price_eth':price_eth})
+    ts_coins_cut['id']=np.repeat(coin_name,len(ts_coins_cut))
+    ts_coins_cut['date']=pd.to_datetime(ts_coins_cut['date'])
+
+    # SCRAPE CURRENT DATA
+    unix_all=[]
+
+    #create date variable
+    for val in datum_range:
+        unix_all=np.append(unix_all,time.mktime(val.timetuple()))
+    url = ' https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest'
+    parameters = {
+    'symbol': symbol,
+    'convert':'USD',
+    }
+    headers = {
+    'Accepts': 'application/json',
+    'X-CMC_PRO_API_KEY': COINMARKET_API_KEY,
+    }
+
+    session = Session()
+    session.headers.update(headers)
+
+    try:
+        response = session.get(url, params=parameters)
+        data_json = json.loads(response.text)
+        #data = json.loads(response.text['data']['quote']['USD'])
+
+    except (ConnectionError, Timeout, TooManyRedirects) as e:
+        print(e)
+
+    # Current data ETH
+    #create date variable
+    for val in datum_range:
+        unix_all=np.append(unix_all,time.mktime(val.timetuple()))
+    url = ' https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest'
+    parameters = {
+    'symbol': 'ETH',
+    'convert':'USD',
+    }
+    headers = {
+    'Accepts': 'application/json',
+    'X-CMC_PRO_API_KEY': COINMARKET_API_KEY,
+    }
+
+    session = Session()
+    session.headers.update(headers)
+
+    try:
+        response = session.get(url, params=parameters)
+        data_json_eth = json.loads(response.text)
+        #data = json.loads(response.text['data']['quote']['USD'])
+
+    except (ConnectionError, Timeout, TooManyRedirects) as e:
+        print(e)
+
+    date=data_json['data'][str(symbol)][0]['quote']['USD']['last_updated']
+    market_cap=data_json['data'][str(symbol)][0]['quote']['USD']['market_cap']
+    total_volumes=data_json['data'][str(symbol)][0]['quote']['USD']['volume_24h']
+    price=data_json['data'][str(symbol)][0]['quote']['USD']['price']
+    price_eth=data_json_eth['data']['ETH'][0]['quote']['USD']['price']
+    # CREATE CURRENT ROW
+    from datetime import date
+    today = date.today()
+    df_today_row=pd.DataFrame({0:['id','date','prices','market_caps','total_vol','price_eth'],1:[coin_name[0],today.strftime('%Y-%m-%d %H:%M:%S'),price,market_cap,total_volumes,price_eth]}).T
+    df_today_row.columns=df_today_row.iloc[0,:]
+    df_today_row=df_today_row.drop(0)
+    ts_coins_cut.to_csv('ts_coins_cut.csv')    
+    return ts_coins_cut, df_today_row
 
 # 2. Macro variables, CLI
 
@@ -208,7 +289,7 @@ def scrape_google_trends(currency, currency_short):
     Hour_end=21
     Minute_end=20
 
-    past=today-datetime.timedelta(days=30)
+    past=today-datetime.timedelta(days=200)
     Day_start = past.strftime("%d")
     Month_start = past.strftime("%m")
     Year_start = past.strftime("%Y")

tools/.chainlit/config.toml

@@ -0,0 +1,84 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Show the prompt playground
+prompt_playground = true
+
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Authorize users to upload files with messages
+multi_modal = true
+
+# Allows user to use speech to text
+[features.speech_to_text]
+    enabled = false
+    # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string
+    # language = "en-US"
+
+[UI]
+# Name of the app and chatbot.
+name = "Chatbot"
+
+# Show the readme while the conversation is empty.
+show_readme_as_default = true
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+
+[meta]
+generated_by = "0.7.700"

tools/crypto_sentiment_analysis_util.py

@@ -0,0 +1,282 @@
+
+import os
+from dotenv import load_dotenv
+from transformers import pipeline
+import os
+import pandas as pd
+from GoogleNews import GoogleNews
+from langchain_openai import ChatOpenAI
+import pandas as pd
+import praw
+from datetime import datetime
+
+load_dotenv()
+
+def fetch_news(stockticker):
+    
+    """ Fetches news articles for a given stock symbol within a specified date range.
+
+    Args:
+    - stockticker (str): Symbol of a particular stock
+    
+    Returns:
+    - list: A list of dictionaries containing stock news. """
+    
+    load_dotenv()
+    days_to_fetch_news = os.environ["DAYS_TO_FETCH_NEWS"]
+
+    googlenews = GoogleNews()
+    googlenews.set_period(days_to_fetch_news)
+    googlenews.get_news(stockticker)
+    news_json=googlenews.get_texts()
+    urls=googlenews.get_links()
+    
+    no_of_news_articles_to_fetch = os.environ["NO_OF_NEWS_ARTICLES_TO_FETCH"]
+    news_article_list = []
+    counter = 0
+    for article in news_json:
+        
+        if(counter >= int(no_of_news_articles_to_fetch)):
+            break
+
+        relevant_info = {
+            'News_Article': article,
+            'URL': urls[counter]
+        }
+        news_article_list.append(relevant_info)
+        counter+=1
+    return news_article_list
+
+def fetch_reddit_news(cryptocurrencyticker):
+    load_dotenv()
+    REDDIT_USER_AGENT= os.environ["REDDIT_USER_AGENT"]
+    REDDIT_CLIENT_ID= os.environ["REDDIT_CLIENT_ID"]
+    REDDIT_CLIENT_SECRET= os.environ["REDDIT_CLIENT_SECRET"]
+    #https://medium.com/geekculture/a-complete-guide-to-web-scraping-reddit-with-python-16e292317a52
+    user_agent = REDDIT_USER_AGENT
+    reddit = praw.Reddit (
+    client_id= REDDIT_CLIENT_ID,
+    client_secret= REDDIT_CLIENT_SECRET,
+    user_agent=user_agent
+    )
+
+    headlines = set ( )
+    for submission in reddit.subreddit('CryptoCurrencyTrading').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CryptoCurrencyTrading').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CryptoCurrencyTrading').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# coinbase
+    for submission in reddit.subreddit('CoinBase').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CoinBase').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CoinBase').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# coingecko
+    for submission in reddit.subreddit('coingecko').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('coingecko').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('coingecko').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# crypto
+    for submission in reddit.subreddit('CryptoCurrency').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CryptoCurrency').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('CryptoCurrency').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# shitcoin
+    for submission in reddit.subreddit('ShitcoinCentral').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('ShitcoinCentral').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('ShitcoinCentral').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# shitcoin
+    for submission in reddit.subreddit('shitcoinmoonshots').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('shitcoinmoonshots').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('shitcoinmoonshots').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+# solana
+    for submission in reddit.subreddit('solana').search(cryptocurrencyticker,time_filter='week'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    if len(headlines)<10:
+        for submission in reddit.subreddit('solana').search(cryptocurrencyticker,time_filter='year'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    if len(headlines)<10:
+        for submission in reddit.subreddit('solana').search(cryptocurrencyticker): #,time_filter='week'):
+            headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+    return headlines
+
+def analyze_sentiment(article):
+    """
+    Analyzes the sentiment of a given news article.
+
+    Args:
+    - news_article (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+    Returns:
+    - dict: A dictionary containing sentiment analysis results.
+    """
+
+    #Analyze sentiment using default model
+    #classifier = pipeline('sentiment-analysis')
+
+    #Analyze sentiment using specific model
+    classifier = pipeline(model='mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis')
+    sentiment_result = classifier(str(article))
+
+    analysis_result = {
+        'News_Article': article,
+        'Sentiment': sentiment_result
+    }
+
+    return analysis_result
+
+
+def generate_summary_of_sentiment(sentiment_analysis_results):
+    
+    
+    news_article_sentiment = str(sentiment_analysis_results)
+    print("News article sentiment : " + news_article_sentiment)
+    
+
+    os.environ["OPENAI_API_KEY"] = os.environ["OPENAI_API_KEY"]
+    model = ChatOpenAI(
+        model="gpt-4o",
+        temperature=0,
+        max_tokens=None,
+        timeout=None,
+        max_retries=2,
+        # api_key="...",  # if you prefer to pass api key in directly instaed of using env vars
+        # base_url="...",
+        # organization="...",
+        # other params...
+    )
+
+    messages=[
+            {"role": "system", "content": "You are a helpful assistant that looks at all news articles with their sentiment, hyperlink and date in front of the article text, the articles MUST be ordered by date!, and generate a summary rationalizing dominant sentiment. At the end of the summary, add URL links with dates for all the articles in the markdown format for streamlit. Make sure the articles as well as the links are ordered descending by Date!!!!!!! Example of adding the URLs: The Check out the links: [link](%s) % url, 2024-03-01. "},
+            {"role": "user", "content": f"News articles and their sentiments: {news_article_sentiment}"}
+    ]
+    response = model.invoke(messages)
+    
+
+    summary = response.content
+    print ("+++++++++++++++++++++++++++++++++++++++++++++++")
+    print(summary)
+    print ("+++++++++++++++++++++++++++++++++++++++++++++++")
+    return summary
+
+
+def plot_sentiment_graph(sentiment_analysis_results):
+    """
+    Plots a sentiment analysis graph 
+
+    Args:
+    - sentiment_analysis_result): (dict): Dictionary containing 'Review Title : Summary', 'Rating', and 'Sentiment' keys.
+
+    Returns:
+    - dict: A dictionary containing sentiment analysis results.
+    """
+    df = pd.DataFrame(sentiment_analysis_results)
+    print(df)
+
+    #Group by Rating, sentiment value count
+    grouped = df['Sentiment'].value_counts()
+
+    sentiment_counts = df['Sentiment'].value_counts()
+
+    # Plotting pie chart
+    # fig = plt.figure(figsize=(5, 3))
+    # plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=140)
+    # plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
+    
+    #Open below when u running this program locally and c
+    #plt.show()
+
+    return sentiment_counts
+
+
+def get_dominant_sentiment (sentiment_analysis_results):
+    """
+    Returns overall sentiment, negative or positive or neutral depending on the count of negative sentiment vs positive sentiment 
+
+    Args:
+    - sentiment_analysis_result): (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+    Returns:
+    - dict: A dictionary containing sentiment analysis results.
+    """
+    df = pd.DataFrame(sentiment_analysis_results)
+
+    # Group by the 'sentiment' column and count the occurrences of each sentiment value
+    sentiment_counts = df['Sentiment'].value_counts().reset_index()
+    sentiment_counts.columns = ['sentiment', 'count']
+    print(sentiment_counts)
+
+    # Find the sentiment with the highest count
+    dominant_sentiment = sentiment_counts.loc[sentiment_counts['count'].idxmax()]
+
+    return dominant_sentiment['sentiment']
+
+#starting point of the program
+if __name__ == '__main__':
+    
+    #fetch stock news
+    news_articles = fetch_news('AAPL')
+
+    analysis_results = []
+    
+    #Perform sentiment analysis for each product review
+    for article in news_articles:
+        sentiment_analysis_result = analyze_sentiment(article['News_Article'])
+
+        # Display sentiment analysis results
+        print(f'News Article: {sentiment_analysis_result["News_Article"]} : Sentiment: {sentiment_analysis_result["Sentiment"]}', '\n')
+
+        result = {
+                    'News_Article': sentiment_analysis_result["News_Article"],
+                    'Sentiment': sentiment_analysis_result["Sentiment"][0]['label']
+                }
+        
+        analysis_results.append(result)
+
+    
+    #Graph dominant sentiment based on sentiment analysis data of reviews
+    dominant_sentiment = get_dominant_sentiment(analysis_results)
+    print(dominant_sentiment)
+    
+    #Plot graph
+    plot_sentiment_graph(analysis_results)
+

tools/data_analyst.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel, Field
+from langchain.tools import BaseTool
+from typing import Optional, Type
+from langchain.tools import StructuredTool
+import yfinance as yf
+from typing import List
+from datetime import datetime,timedelta
+from pycoingecko import CoinGeckoAPI
+cg = CoinGeckoAPI()
+
+def data_analyst_tools():
+    def get_crypto_price(cryptocurrencyticker: str) -> str:
+        current_data=cg.get_price(ids=cryptocurrencyticker, vs_currencies='usd',include_market_cap='true', include_24hr_vol='true',include_last_updated_at='true')
+        return str(current_data)
+
+    class CryptoPriceCheckInput(BaseModel):
+        """Input for Crypto price check."""
+        Cryptoticker: str = Field(..., description="Ticker symbol for Crypto or index")
+
+    class CryptoPriceTool(BaseTool):
+        name = "get_crypto_price"
+        description = "Useful for when you need to find out the price of Cryptocurrency. You should input the Crypto ticker used on the Coingecko API"
+        """Input for Cryptocurrency price check."""
+        Cryptoticker: str = Field(..., description="Ticker symbol for Crypto or index")
+        def _run(self, Cryptoticker: str):
+            # print("i'm running")
+            price_response = get_crypto_price(Cryptoticker)
+
+            return str(price_response)
+
+        def _arun(self, Cryptoticker: str):
+            raise NotImplementedError("This tool does not support async")
+        args_schema: Optional[Type[BaseModel]] = CryptoPriceCheckInput
+
+
+    tools_data_analyst = [StructuredTool.from_function(
+            func=CryptoPriceTool,
+            args_schema=CryptoPriceCheckInput,
+            description="Function to get current Crypto prices.",
+        ),
+    ]
+    return tools_data_analyst

tools/df_history.csv

@@ -0,0 +1,63 @@
+Date,Open,High,Low,Close,Volume,Dividends,Stock Splits,stockticker
+2024-04-22 00:00:00-04:00,399.3596471827562,402.1246793258712,395.03745670124425,400.2380676269531,20286900,0.0,0.0,MSFT
+2024-04-23 00:00:00-04:00,403.51216021293357,407.4650522060062,402.33429210205253,406.836181640625,15734500,0.0,0.0,MSFT
+2024-04-24 00:00:00-04:00,408.82258607970806,411.72735028943725,406.0475926794115,408.323486328125,15065300,0.0,0.0,MSFT
+2024-04-25 00:00:00-04:00,393.32054400787314,399.1700088625431,387.3313470651067,398.321533203125,40586500,0.0,0.0,MSFT
+2024-04-26 00:00:00-04:00,411.4279132786848,412.25640548421114,405.02945064216703,405.58843994140625,29694700,0.0,0.0,MSFT
+2024-04-29 00:00:00-04:00,404.52035539531056,405.5884361925186,398.4712687423875,401.5257568359375,19582100,0.0,0.0,MSFT
+2024-04-30 00:00:00-04:00,400.76710737423014,401.4359144425664,388.4693126899027,388.6289978027344,28781400,0.0,0.0,MSFT
+2024-05-01 00:00:00-04:00,391.9030904630616,400.9967037344784,389.6072438016868,394.2289123535156,23562500,0.0,0.0,MSFT
+2024-05-02 00:00:00-04:00,396.94401914412265,399.20992105581087,393.9394288835304,397.1236877441406,17709400,0.0,0.0,MSFT
+2024-05-03 00:00:00-04:00,401.55570709720826,406.4169339510819,401.13644988960164,405.9278259277344,17446700,0.0,0.0,MSFT
+2024-05-06 00:00:00-04:00,408.024048156178,413.1847226485525,405.63833666603693,412.7954406738281,16996600,0.0,0.0,MSFT
+2024-05-07 00:00:00-04:00,413.91342570011614,413.92341744357753,408.35344694069664,408.6029968261719,20018200,0.0,0.0,MSFT
+2024-05-08 00:00:00-04:00,407.4351142805277,411.48780192255407,405.97772103822234,409.80084228515625,11792300,0.0,0.0,MSFT
+2024-05-09 00:00:00-04:00,409.8307875446534,411.97691043744567,408.363433019907,411.57763671875,14689700,0.0,0.0,MSFT
+2024-05-10 00:00:00-04:00,412.1965086797442,414.6321179246016,411.05854661467066,413.9932556152344,13402300,0.0,0.0,MSFT
+2024-05-13 00:00:00-04:00,417.2573820335048,417.5967662074119,410.08032520369875,412.97509765625,15440200,0.0,0.0,MSFT
+2024-05-14 00:00:00-04:00,411.2781631216723,416.73831581889846,410.8090081198034,415.80999755859375,15109300,0.0,0.0,MSFT
+2024-05-15 00:00:00-04:00,417.8999938964844,423.80999755859375,417.2699890136719,423.0799865722656,22239500,0.75,0.0,MSFT
+2024-05-16 00:00:00-04:00,421.79998779296875,425.4200134277344,420.3500061035156,420.989990234375,17530100,0.0,0.0,MSFT
+2024-05-17 00:00:00-04:00,422.5400085449219,422.9200134277344,418.0299987792969,420.2099914550781,15352200,0.0,0.0,MSFT
+2024-05-20 00:00:00-04:00,420.2099914550781,426.7699890136719,419.989990234375,425.3399963378906,16272100,0.0,0.0,MSFT
+2024-05-21 00:00:00-04:00,426.8299865722656,432.9700012207031,424.8500061035156,429.0400085449219,21453300,0.0,0.0,MSFT
+2024-05-22 00:00:00-04:00,430.0899963378906,432.4100036621094,427.1300048828125,430.5199890136719,18073700,0.0,0.0,MSFT
+2024-05-23 00:00:00-04:00,432.9700012207031,433.6000061035156,425.4200134277344,427.0,17211700,0.0,0.0,MSFT
+2024-05-24 00:00:00-04:00,427.19000244140625,431.05999755859375,424.4100036621094,430.1600036621094,11845800,0.0,0.0,MSFT
+2024-05-28 00:00:00-04:00,429.6300048828125,430.82000732421875,426.6000061035156,430.32000732421875,15718000,0.0,0.0,MSFT
+2024-05-29 00:00:00-04:00,425.69000244140625,430.94000244140625,425.69000244140625,429.1700134277344,15517100,0.0,0.0,MSFT
+2024-05-30 00:00:00-04:00,424.29998779296875,424.29998779296875,414.239990234375,414.6700134277344,28424800,0.0,0.0,MSFT
+2024-05-31 00:00:00-04:00,416.75,416.75,404.510009765625,415.1300048828125,47995300,0.0,0.0,MSFT
+2024-06-03 00:00:00-04:00,415.5299987792969,416.42999267578125,408.9200134277344,413.5199890136719,17484700,0.0,0.0,MSFT
+2024-06-04 00:00:00-04:00,412.42999267578125,416.44000244140625,409.67999267578125,416.07000732421875,14348900,0.0,0.0,MSFT
+2024-06-05 00:00:00-04:00,417.80999755859375,424.0799865722656,416.29998779296875,424.010009765625,16988000,0.0,0.0,MSFT
+2024-06-06 00:00:00-04:00,424.010009765625,425.30999755859375,420.5799865722656,424.5199890136719,14861300,0.0,0.0,MSFT
+2024-06-07 00:00:00-04:00,426.20001220703125,426.2799987792969,423.0,423.8500061035156,13621700,0.0,0.0,MSFT
+2024-06-10 00:00:00-04:00,424.70001220703125,428.0799865722656,423.8900146484375,427.8699951171875,14003000,0.0,0.0,MSFT
+2024-06-11 00:00:00-04:00,425.4800109863281,432.82000732421875,425.25,432.67999267578125,14551100,0.0,0.0,MSFT
+2024-06-12 00:00:00-04:00,435.32000732421875,443.3999938964844,433.25,441.05999755859375,22366200,0.0,0.0,MSFT
+2024-06-13 00:00:00-04:00,440.8500061035156,443.3900146484375,439.3699951171875,441.5799865722656,15960600,0.0,0.0,MSFT
+2024-06-14 00:00:00-04:00,438.2799987792969,443.1400146484375,436.7200012207031,442.57000732421875,13582000,0.0,0.0,MSFT
+2024-06-17 00:00:00-04:00,442.5899963378906,450.94000244140625,440.7200012207031,448.3699951171875,20790000,0.0,0.0,MSFT
+2024-06-18 00:00:00-04:00,449.7099914550781,450.1400146484375,444.8900146484375,446.3399963378906,17112500,0.0,0.0,MSFT
+2024-06-20 00:00:00-04:00,446.29998779296875,446.5299987792969,441.2699890136719,445.70001220703125,19877400,0.0,0.0,MSFT
+2024-06-21 00:00:00-04:00,447.3800048828125,450.5799865722656,446.510009765625,449.7799987792969,34486200,0.0,0.0,MSFT
+2024-06-24 00:00:00-04:00,449.79998779296875,452.75,446.4100036621094,447.6700134277344,15913700,0.0,0.0,MSFT
+2024-06-25 00:00:00-04:00,448.25,451.4200134277344,446.75,450.95001220703125,16747500,0.0,0.0,MSFT
+2024-06-26 00:00:00-04:00,449.0,453.6000061035156,448.19000244140625,452.1600036621094,16507000,0.0,0.0,MSFT
+2024-06-27 00:00:00-04:00,452.17999267578125,456.1700134277344,451.7699890136719,452.8500061035156,14806300,0.0,0.0,MSFT
+2024-06-28 00:00:00-04:00,453.07000732421875,455.3800048828125,446.4100036621094,446.95001220703125,28362300,0.0,0.0,MSFT
+2024-07-01 00:00:00-04:00,448.6600036621094,457.3699951171875,445.6600036621094,456.7300109863281,17662800,0.0,0.0,MSFT
+2024-07-02 00:00:00-04:00,453.20001220703125,459.5899963378906,453.1099853515625,459.2799987792969,13979800,0.0,0.0,MSFT
+2024-07-03 00:00:00-04:00,458.19000244140625,461.0199890136719,457.8800048828125,460.7699890136719,9932800,0.0,0.0,MSFT
+2024-07-05 00:00:00-04:00,459.6099853515625,468.3500061035156,458.9700012207031,467.55999755859375,16000300,0.0,0.0,MSFT
+2024-07-08 00:00:00-04:00,466.54998779296875,467.70001220703125,464.4599914550781,466.239990234375,12962300,0.0,0.0,MSFT
+2024-07-09 00:00:00-04:00,467.0,467.3299865722656,458.0,459.5400085449219,17207200,0.0,0.0,MSFT
+2024-07-10 00:00:00-04:00,461.2200012207031,466.4599914550781,458.8599853515625,466.25,18196100,0.0,0.0,MSFT
+2024-07-11 00:00:00-04:00,462.9800109863281,464.7799987792969,451.54998779296875,454.70001220703125,23111200,0.0,0.0,MSFT
+2024-07-12 00:00:00-04:00,454.3299865722656,456.3599853515625,450.6499938964844,453.54998779296875,16311300,0.0,0.0,MSFT
+2024-07-15 00:00:00-04:00,453.29998779296875,457.260009765625,451.42999267578125,453.9599914550781,14429400,0.0,0.0,MSFT
+2024-07-16 00:00:00-04:00,454.2200012207031,454.29998779296875,446.6600036621094,449.5199890136719,17175700,0.0,0.0,MSFT
+2024-07-17 00:00:00-04:00,442.5899963378906,444.8500061035156,439.17999267578125,443.5199890136719,21778000,0.0,0.0,MSFT
+2024-07-18 00:00:00-04:00,444.3399963378906,444.6499938964844,434.3999938964844,440.3699951171875,20794800,0.0,0.0,MSFT
+2024-07-19 00:00:00-04:00,433.1000061035156,441.1400146484375,432.0,437.1099853515625,20862400,0.0,0.0,MSFT

tools/df_with_forecast.csv

@@ -0,0 +1,113 @@
+,id,prices,market_caps,total_vol,CLI,CPI,Employment,id,google_trend,GSPC,GC=F,EURUSD,TNX
+2023-01-31,,,,,107.893,,,,,,,,
+2023-02-28,,,,,,,,,,,,,
+2023-03-31,,,,,,,,,,,,,
+2023-04-30,,,,,108.87,,,,,,,,
+2023-05-31,,,,,,,,,,,,,
+2023-06-30,,,,,,,,,,,,,
+2023-07-31,,,,,108.931,,,,,,,,
+2023-08-31,,,,,,,,,,,,,
+2023-09-30,,,,,,,,,,,,,
+2023-10-31,,,,,105.3285,,,,,,,,
+2023-11-30,,,,,101.961,,,,,,,,
+2023-12-31,,,,,76.021485,,,,,,,,
+2024-01-01,,,,,,308.417,161152.0,,,,,,
+2024-01-31,,,,,82.796104,,,,,,,,
+2024-02-01,,,,,,310.326,160968.0,,,,,,
+2024-02-29,,,,,50.859505000000006,,,,,,,,
+2024-03-01,,,,,,312.332,161466.0,,,,,,
+2024-03-31,,,,,71.54424900000001,,,,,,,,
+2024-04-01,,,,,,313.548,161491.0,,,,,,
+2024-04-30,,,,,70.99926128571428,,,,,,,,
+2024-05-01,,,,,,314.069,161083.0,,,,,,
+2024-05-06,ethereum,3114.4007005303224,374072475993.8121,11127068946.811003,,,,,,5142.419921875,2322.800048828125,1.0758124589920044,4.486999988555908
+2024-05-07,ethereum,3062.1337546278614,367763583617.18,12212682358.435501,,,,,,5187.2001953125,2324.300048828125,1.0767507553100586,4.4730000495910645
+2024-05-08,ethereum,2999.4869525045415,360243594935.94305,11179447639.771477,,,,,,5168.97998046875,2313.60009765625,1.0750491619110107,4.484000205993652
+2024-05-09,ethereum,3003.5642861321066,360831840100.2795,10896607994.801586,,,,,,5189.02978515625,2310.699951171875,1.0746910572052002,4.515999794006348
+2024-05-10,ethereum,2966.441885585809,356315503171.5778,11384672900.217531,,,,,,5225.490234375,2367.300048828125,1.078515887260437,4.4770002365112305
+2024-05-11,ethereum,2916.0617572305187,350195653103.13715,9198771437.235367,,,,,,,,,
+2024-05-12,ethereum,2924.2519718055137,351264187115.9079,5460016379.689179,,,,,,,,,
+2024-05-13,ethereum,2945.101626776707,353629048507.87177,11486452597.64263,,,,,,5233.080078125,2358.300048828125,1.077040672302246,4.486999988555908
+2024-05-14,ethereum,2900.6563297650755,348385655037.8947,11542903142.535707,,,,,,5221.10009765625,2336.0,1.079354166984558,4.480999946594238
+2024-05-15,ethereum,2963.759315698215,355948495364.97504,11917611608.681692,,,,,,5263.259765625,2361.60009765625,1.0814552307128906,4.418000221252441
+2024-05-16,ethereum,2973.3231927105603,357110705527.3933,12982589615.756212,,,,,,5310.06982421875,2389.5,1.0889805555343628,4.329999923706055
+2024-05-17,ethereum,3055.768061630982,366942477718.2434,13075157085.833336,,,,,,5303.10009765625,2380.699951171875,1.0867793560028076,4.395999908447266
+2024-05-18,ethereum,3116.6010226358244,374444822902.376,10245527976.19185,,,,,,,,,
+2024-05-19,ethereum,3089.6899077803787,371102827471.68726,7186818177.12275,,,,,,,,,
+2024-05-20,ethereum,3275.1726235398733,393064227949.9598,14861247848.577646,,,,,,5305.35009765625,2415.800048828125,1.0875475406646729,4.421999931335449
+2024-05-21,ethereum,3736.779369147562,449277769707.8145,39114320451.96952,,,,,,5298.68994140625,2429.5,1.086082935333252,4.429999828338623
+2024-05-22,ethereum,3743.6797798618477,449904838634.0898,27177492763.312775,,,,,,5319.27978515625,2417.5,1.0855170488357544,4.453000068664551
+2024-05-23,ethereum,3802.195186351903,456440243805.41815,31490184022.903027,,,,,,5340.259765625,2371.199951171875,1.0825204849243164,4.418000221252441
+2024-05-24,ethereum,3716.584005854382,446214223854.924,36595327650.52252,,,,,,5281.4501953125,2342.60009765625,1.0812681913375854,4.488999843597412
+2024-05-25,ethereum,3747.9139284087246,450191345047.8288,12320451102.246416,,,,,,,,,
+2024-05-26,ethereum,3822.9062956505663,459222383843.30835,10446532594.41313,,,,,,,,,
+2024-05-27,ethereum,3904.4531708423287,468888345618.59467,15875563017.846474,,,,,,,,,
+2024-05-28,ethereum,3850.4555712911824,462421783973.1804,16981798566.54435,,,,,,5315.91015625,2336.89990234375,1.0847634077072144,4.453000068664551
+2024-05-29,ethereum,3790.3890178850775,455720009606.71265,16291087718.980705,,,,,,5278.72998046875,2340.300048828125,1.0861891508102417,4.565999984741211
+2024-05-30,ethereum,3760.0562456291614,451909655831.15967,14241005007.401228,,,,,,5259.77001953125,2336.89990234375,1.0851871967315674,4.593999862670898
+2024-05-31,ethereum,3766.348666657115,452613341252.82587,13233598539.854492,74.31149225,,,,,5243.2099609375,2344.10009765625,1.0801819562911987,4.552000045776367
+2024-06-01,ethereum,3797.773661763607,456183752606.3188,10134684638.033493,,314.175,161199.0,,,,,,
+2024-06-02,ethereum,3789.4643581084024,455395042781.10913,8704992695.338041,,,,,,,,,
+2024-06-03,ethereum,3792.876939393309,455583728457.67914,11345758405.200346,,,,,,5297.14990234375,2322.60009765625,1.0835410356521606,4.4730000495910645
+2024-06-04,ethereum,3791.003611630613,455454864609.9826,11583124383.448898,,,,,,5278.240234375,2347.5,1.0853049755096436,4.361000061035156
+2024-06-05,ethereum,3826.985885858965,459614958452.3508,12970952878.62752,,,,,,5314.47998046875,2326.39990234375,1.0907385349273682,4.326000213623047
+2024-06-06,ethereum,3831.1242540234693,460276696407.6153,12084914430.50892,,,,,,5357.7998046875,2355.0,1.0881866216659546,4.301000118255615
+2024-06-07,ethereum,3759.215714495715,451768670762.0454,13410143841.10996,,,,,,5343.81005859375,2379.89990234375,1.087453007698059,4.296999931335449
+2024-06-08,ethereum,3684.6113824590207,442729750197.81464,12328054405.91001,,,,,,,,,
+2024-06-09,ethereum,3693.5688944666886,443758507298.5523,6434938580.374867,,,,,,,,,
+2024-06-10,ethereum,3669.804414640858,441385340635.06006,7678615033.327232,,,,,,5341.22021484375,2290.60009765625,1.0894432067871094,4.453000068664551
+2024-06-11,ethereum,3511.552216421044,421773678119.8096,16295040761.774801,,,,,,5353.0,2300.0,1.0778882503509521,4.435999870300293
+2024-06-12,ethereum,3559.325489288785,427430429412.461,16146566925.235643,,,,,,5409.1298828125,2314.89990234375,1.0764262676239014,4.388000011444092
+2024-06-13,ethereum,3487.56598378508,418921286187.6116,16102374829.54766,,,,,,5441.93017578125,2309.39990234375,1.0740330219268799,4.309999942779541
+2024-06-14,ethereum,3477.487513961109,417925411247.93335,13751574508.884224,,,,,,5424.080078125,2307.0,1.0813149213790894,4.196000099182129
+2024-06-15,ethereum,3547.2656444381705,426052557357.5137,14435000601.871336,,,,,,,,,
+2024-06-16,ethereum,3581.554801676437,430252401136.91156,9265313569.748865,,,,,,,,,
+2024-06-17,ethereum,3521.540033404755,423525318255.71875,14792637405.883486,,,,,,5431.10986328125,2320.199951171875,1.0738831758499146,4.264999866485596
+2024-06-18,ethereum,3443.057015401169,420823437646.33813,20647300792.256817,,,,,,5476.14990234375,2311.800048828125,1.0702168941497803,4.2870001792907715
+2024-06-19,ethereum,3548.6088199945443,433926242387.04596,17174770852.62574,,,,,,,,,
+2024-06-20,ethereum,3544.840023868359,433416267378.3272,14431531886.666199,,,,,,5499.990234375,2328.89990234375,1.0740677118301392,4.23799991607666
+2024-06-21,ethereum,3501.4844259897077,428154107816.4788,15591212529.32492,,,,,,5466.77001953125,2331.199951171875,1.0742292404174805,4.223999977111816
+2024-06-22,ethereum,3498.401912499916,427797791720.6712,10074881877.914883,,,,,,,,,
+2024-06-23,ethereum,3471.25661843264,424559563614.4485,6991292722.007789,,,,,,,,,
+2024-06-24,ethereum,3330.9914608412787,405609291184.5871,18078022125.09135,,,,,,5459.580078125,2323.300048828125,1.0751532316207886,4.275000095367432
+2024-06-25,ethereum,3388.6552364887525,407290827369.29706,17480780190.891705,,,,,,5460.72998046875,2324.39990234375,1.0706409215927124,4.2170000076293945
+2024-06-26,ethereum,3379.6384972292053,406200008420.7241,11068675554.45744,,,,,,5460.7099609375,2307.89990234375,1.0687757730484009,4.284999847412109
+2024-06-27,ethereum,3422.9987958226375,411321369785.6611,11381676501.223251,,,,,,5473.58984375,2296.800048828125,1.0732723474502563,4.327000141143799
+2024-06-28,ethereum,3409.9899950655326,410020829397.1302,12108738591.606865,,,,,,5488.47998046875,2325.39990234375,1.0709619522094727,4.303999900817871
+2024-06-29,ethereum,3382.1846820729265,406488247465.7656,8118319341.85326,,,,,,,,,
+2024-06-30,ethereum,3405.653616859288,409202194429.17535,6586293113.04436,74.519282,,,,,,,,
+2024-07-01,ethereum,3466.7870078346746,416633403003.7347,10972345968.355255,,,161266.0,ethereum,16.81,5471.080078125,2323.800048828125,1.068010926246643,4.423999786376953
+2024-07-02,ethereum,3425.9867093877297,411805426012.3614,9347179358.150301,,,,ethereum,13.69,5461.83984375,2330.699951171875,1.0708472728729248,4.441999912261963
+2024-07-03,ethereum,3305.863374711538,397336350754.1616,12936936280.4328,,,,ethereum,18.49,5507.43994140625,2330.89990234375,1.0735257863998413,4.426000118255615
+2024-07-04,ethereum,3141.9137606677323,377789953332.375,19762357071.82109,,,,ethereum,18.49,,,,
+2024-07-05,ethereum,2942.873381316971,353947760154.63116,30371642360.497025,,,,ethereum,33.64,5537.91015625,2354.89990234375,1.0737102031707764,4.330999851226807
+2024-07-06,ethereum,3026.0221393875127,363716353821.9705,14095786002.87394,,,,ethereum,17.64,,,,
+2024-07-07,ethereum,2980.329795932314,358196417772.35425,10109122874.859922,,,,ethereum,12.959999999999999,,,,
+2024-07-08,ethereum,2995.42249679924,359848515705.2204,18656672090.05734,,,,ethereum,22.09,5572.75,2381.699951171875,1.0748412609100342,4.306000232696533
+2024-07-09,ethereum,3067.945362448234,368790204798.86127,17265940071.85487,,,,ethereum,18.49,5584.240234375,2363.10009765625,1.0793308019638062,4.300000190734863
+2024-07-10,ethereum,3103.355851261978,373109735757.489,14135312863.462542,,,,ethereum,18.49,5591.259765625,2366.300048828125,1.082602620124817,4.2779998779296875
+2024-07-11,ethereum,3120.2511152159645,375003912446.32135,14601771425.99495,,,,ethereum,20.25,5635.2099609375,2378.699951171875,1.0830950736999512,4.288000106811523
+2024-07-12,ethereum,3106.665060608468,373479677453.21234,13272564100.61316,,,,ethereum,14.44,5590.759765625,2399.800048828125,1.0814785957336426,4.205999851226807
+2024-07-13,ethereum,3156.7229997808245,379336539640.4357,9099839817.153015,,,,ethereum,13.69,,,,
+2024-07-14,ethereum,3214.951666958399,386450506355.89264,9399920281.501238,,,,ethereum,14.44,,,,
+2024-07-15,ethereum,3390.5558977346586,407705698434.75323,14682856047.608225,,,,ethereum,36.0,5638.16015625,2430.0,1.083329677581787,4.236999988555908
+2024-07-16,ethereum,3439.2531522603167,413315389170.71234,20249821643.173527,,,,ethereum,33.64,5644.08984375,2427.39990234375,1.0872756242752075,4.175000190734863
+2024-07-17,ethereum,3445.9301630097125,414204244128.5122,18316846571.47861,,,,ethereum,29.160000000000004,5610.06982421875,2472.89990234375,1.0885539054870605,4.178999900817871
+2024-07-18,ethereum,3428.483712881156,412151252125.53094,14880766047.388323,,,,ethereum,27.04,5608.56005859375,2466.0,1.0901559591293335,4.183000087738037
+2024-07-19,ethereum,3459.866194068627,415830197917.89233,15874038814.582893,,,,ethereum,26.01,5543.3701171875,2418.800048828125,1.0902509689331055,4.224999904632568
+2024-07-20,ethereum,3506.166559401983,421506391873.4467,12819209095.177147,,,,ethereum,24.009999999999998,,,,
+2024-07-21,ethereum,3505.198785199442,421479666822.44586,9734735125.637077,,,,ethereum,24.009999999999998,,,,
+2024-07-22,ethereum,3478.5398389521833,418248333345.1597,17165553086.473969,,,,ethereum,43.56,5544.5400390625,2402.10009765625,1.094020128250122,4.224999904632568
+2024-07-23,ethereum,3471.4537045282686,417282646136.00616,22247562224.91563,,,,ethereum,100.0,5565.2998046875,2395.800048828125,1.0901559591293335,4.236999988555908
+2024-07-24,ethereum,3389.34554552627,407555827602.13794,19555399218.051266,,,,ethereum,53.29,5505.83984375,2421.0,1.0899182558059692,4.229000091552734
+2024-07-25,ethereum,3166.8674273376732,380757485120.10834,23309182458.392136,,,,ethereum,49.0,5428.7001953125,2365.5,1.0889805555343628,4.205999851226807
+2024-07-26,ethereum,3256.836248537211,391618277681.34503,18543581673.430332,,,,ethereum,32.489999999999995,5433.669921875,2368.699951171875,1.0850694179534912,4.244999885559082
+2024-07-27,ethereum,3268.634820892746,393058948301.01434,12350215232.854982,,,,ethereum,26.01,,,,
+2024-07-28,ethereum,3261.358165252211,392191417408.00104,11791845044.016218,,,,ethereum,22.09,,,,
+2024-07-29,ethereum,3339.1396964471446,401594098625.7107,13845546119.727053,,,,ethereum,30.250000000000004,5476.5498046875,2377.300048828125,1.0839520692825317,4.163000106811523
+2024-07-30,ethereum,3307.2248771443924,397741773052.1379,15364998924.01399,,,,ethereum,21.16,5478.72998046875,2380.89990234375,1.0851283073425293,4.173999786376953
+2024-07-31,ethereum,3281.768911864367,394601659805.1198,14197119142.220865,99.00630000000001,,,ethereum,19.36,5505.58984375,2407.10009765625,1.0853756666183472,4.127999782562256
+2024-08-01,ethereum,3164.2867044555733,380493843585.9209,19172882534.0517,,,,ethereum,45.0,5537.83984375,2446.699951171875,1.081946611404419,4.052999973297119
+2024-08-02,ethereum,3067.1887722202587,368925501105.263,19531566013.812,,,,ethereum,37.309999999999995,5376.6298828125,2490.800048828125,1.0816072225570679,3.940000057220459
+2024-08-03,ethereum,2946.763174245429,354325094002.6969,19461236907.28371,,,,ethereum,35.6,,,,
+2024-08-04,ethereum,2911.61,350078479982.3696,13827666339.828363,,,,ethereum,40.42,,,,

tools/stock_sentiment_evalutor.py

@@ -0,0 +1,261 @@
+from transformers import pipeline
+from alpaca_trade_api import REST
+import os
+from dotenv import load_dotenv
+from datetime import datetime
+import pandas as pd
+import matplotlib.pyplot as plt
+from datetime import date, timedelta
+from pydantic import BaseModel, Field
+from langchain.tools import BaseTool
+from typing import Optional, Type
+from langchain.tools import StructuredTool
+
+
+def sentimental_analysis_tools():
+
+    class AlpacaNewsFetcher:
+        """
+        A class for fetching news articles related to a specific stock from Alpaca API.
+
+        Attributes:
+        - api_key (str): Alpaca API key for authentication.
+        - api_secret (str): Alpaca API secret for authentication.
+        - rest_client (alpaca_trade_api.REST): Alpaca REST API client.
+        """
+
+        def __init__(self):
+            """
+            Initializes the AlpacaNewsFetcher object.
+
+            Args:
+            - api_key (str): Alpaca API key for authentication.
+            - api_secret (str): Alpaca API secret for authentication.
+            """
+            load_dotenv()
+            self.api_key = os.environ["ALPACA_API_KEY"]
+            self.api_secret = os.environ["ALPACA_SECRET"]
+            self.rest_client = REST(self.api_key, self.api_secret)
+
+            #No of news articles to fetch for the input stock ticker.
+            self.no_of_newsarticles_to_fetch = os.environ["NO_OF_NEWSARTICLES_TO_FETCH"]
+
+            #No of days to fetch news articles for
+            self.no_of_days = os.environ["NO_OF_DAYS_TO_FETCH_NEWS_ARTICLES"]
+
+
+        def fetch_news(self, stockticker):
+            """
+            Fetches news articles for a given stock symbol within a specified date range.
+
+            Args:
+            - stockticker (str): Stock symbol for which news articles are to be fetched (e.g., "AAPL").
+
+            Returns:
+            - list: A list of dictionaries containing relevant information for each news article.
+            """
+
+            #Date range for which to get the news
+            start_date = date.today()
+            end_date = date.today() - timedelta(self.no_of_days)
+
+            news_articles = self.rest_client.get_news(stockticker, start_date, end_date, limit=self.no_of_newsarticles_to_fetch )
+            formatted_news = []
+
+            for article in news_articles:
+                summary = article.summary
+                title = article.headline
+                timestamp = article.created_at
+
+                relevant_info = {
+                    'timestamp': timestamp,
+                    'title': title,
+                    'summary': summary
+                }
+
+                formatted_news.append(relevant_info)
+
+            return formatted_news
+    
+    
+    class NewsSentimentAnalysis:
+        """
+        A class for sentiment analysis of news articles using the Transformers library.
+
+        Attributes:
+            - classifier (pipeline): Sentiment analysis pipeline from Transformers.
+        """
+
+        def __init__(self):
+            """
+            Initializes the NewsSentimentAnalysis object.
+            """
+            self.classifier = pipeline('sentiment-analysis')
+            
+
+        def analyze_sentiment(self, news_article):
+            """
+            Analyzes the sentiment of a given news article.
+
+            Args:
+            - news_article (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+            Returns:
+            - dict: A dictionary containing sentiment analysis results.
+            """
+            summary = news_article['summary']
+            title = news_article['title']
+            timestamp = news_article['timestamp']
+
+            relevant_text = summary + title
+            sentiment_result = self.classifier(relevant_text)
+
+            analysis_result = {
+                'timestamp': timestamp,
+                'title': title,
+                'summary': summary,
+                'sentiment': sentiment_result
+            }
+
+            return analysis_result
+    
+        def plot_sentiment_graph(self, sentiment_analysis_result):
+            """
+            Plots a sentiment analysis graph 
+
+            Args:
+            - sentiment_analysis_result): (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+            Returns:
+            - dict: A dictionary containing sentiment analysis results.
+            """
+            df = pd.DataFrame(sentiment_analysis_result)
+            df['Timestamp'] = pd.to_datetime(df['Timestamp'])
+            df['Date'] = df['Timestamp'].dt.date
+
+            #Group by Date, sentiment value count
+            grouped = df.groupby(by='Date')['Sentiment'].value_counts()
+
+            grouped.plot.pie()
+
+        def get_dominant_sentiment (self, sentiment_analysis_result):
+            """
+            Returns overall sentiment, negative or positive or neutral depending on the count of negative sentiment vs positive sentiment 
+
+            Args:
+            - sentiment_analysis_result): (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+            Returns:
+            - dict: A dictionary containing sentiment analysis results.
+            """
+            df = pd.DataFrame(sentiment_analysis_result)
+            df['Timestamp'] = pd.to_datetime(df['Timestamp'])
+            df['Date'] = df['Timestamp'].dt.date
+
+            #Group by Date, sentiment value count
+            grouped = df.groupby(by='Date')['Sentiment'].value_counts()
+            df = pd.DataFrame(list(grouped.items()), columns=['Sentiment', 'count'])
+            df['date'] = df['Sentiment'].apply(lambda x: x[0])
+            df['sentiment'] = df['Sentiment'].apply(lambda x: x[1])
+            df.drop('Sentiment', axis=1, inplace=True)
+            result = df.groupby('sentiment')['count'].sum().reset_index()
+            
+            # Determine the sentiment with the most count
+            dominant_sentiment = result.loc[result['count'].idxmax()]
+
+            return dominant_sentiment
+
+
+    #Function to get the stock sentiment
+    def get_stock_sentiment(stockticker: str):
+ 
+        #Initialize AlpacaNewsFetcher, a class for fetching news articles related to a specific stock from Alpaca API.
+        news_fetcher = AlpacaNewsFetcher()
+
+
+        # Fetch news (contains - title of the news, timestamp and summary) for specified stocksticker
+        news_data = news_fetcher.fetch_news(stockticker)
+
+        # Initialize the NewsSentimentAnalysis object
+        news_sentiment_analyzer = NewsSentimentAnalysis()
+        analysis_result = []
+       
+        # Assume 'news_data' is a list of news articles (each as a dictionary), analyze sentiment of each news
+        for article in news_data:
+            sentiment_analysis_result = news_sentiment_analyzer.analyze_sentiment(article)
+
+            # Display sentiment analysis results
+            print(f'Timestamp: {sentiment_analysis_result["timestamp"]}, '
+                f'Title: {sentiment_analysis_result["title"]}, '
+                f'Summary: {sentiment_analysis_result["summary"]}')
+
+            print(f'Sentiment: {sentiment_analysis_result["sentiment"]}', '\n')
+
+            result = {
+                    'Timestamp': sentiment_analysis_result["timestamp"],
+                    'News- Title:Summar': sentiment_analysis_result["title"] + sentiment_analysis_result["summary"],
+                    'Sentiment': sentiment_analysis_result["sentiment"][0]['label']
+                }
+            analysis_result.append(result)
+
+            #Extracting timestamp of article and sentiment of article for graphing
+            """  result_for_graph = {
+                    'Timestamp': sentiment_analysis_result["timestamp"],
+                    'Sentiment': sentiment_analysis_result["sentiment"][0]['label']
+                }
+             
+            analysis_result.append(result_for_graph)
+            """
+
+        #Get dominant sentiment
+        dominant_sentiment = news_sentiment_analyzer.get_dominant_sentiment(sentiment_analysis_result)
+
+        #Build response string for news sentiment
+        output_string = ""
+        for result in analysis_result:
+            output_string = output_string + f'{result["Timestamp"]} : {result["News- Title:Summary"]} : {result["Sentiment"]}' + '\n'
+        
+        final_result = {
+                'Sentiment-analysis-result' : output_string,
+                'Dominant-sentiment' : dominant_sentiment['sentiment']
+        }
+
+        return final_result    
+
+
+    class StockSentimentCheckInput(BaseModel):
+        """Input for Stock price check."""
+        stockticker: str = Field(..., description="Ticker symbol for stock or index")
+
+    class StockSentimentAnalysisTool(BaseTool):
+        name = "get_stock_sentiment"
+        description = """Useful for finding sentiment of stock, based on published news articles. 
+                        Fetches configured number of news items for the sentiment, 
+                        determines sentiment of each news items and then returns 
+                        List of sentiment analysit result & domainant sentiment of the news
+                        """
+        
+        """Input for Stock sentiment analysis."""
+        stockticker: str = Field(..., description="Ticker symbol for stock or index")
+        def _run(self, stockticker: str):
+            # print("i'm running")
+            sentiment_response = get_stock_sentiment(stockticker)
+            print("++++++++++++++++++++++++++++++++++++++++++++++++++++++")
+            print(str(sentiment_response))
+            print("++++++++++++++++++++++++++++++++++++++++++++++++++++++")
+
+            return sentiment_response
+
+        def _arun(self, stockticker: str):
+            raise NotImplementedError("This tool does not support async")
+        
+        args_schema: Optional[Type[BaseModel]] = StockSentimentCheckInput
+
+
+    tools_sentiment_analyst = [StructuredTool.from_function(
+            func=StockSentimentAnalysisTool,
+            args_schema=StockSentimentCheckInput,
+            description="Function to get stock sentiment.",
+        )
+    ]
+    return tools_sentiment_analyst