Upload 23 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+.env
+a1.py
+app.ipynb
+/myenv

README.md

@@ -1,12 +1,59 @@
----
-title: Sql Chat Bot
-emoji: 📉
-colorFrom: indigo
-colorTo: purple
-sdk: streamlit
-sdk_version: 1.35.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# SQL Chatbot for Easy Data Extraction
+
+This repository provides a user-friendly SQL chatbot that simplifies extracting information from your tabular files. Leverage the power of OpenAI's language processing and LangChain's SQL generation capabilities to perform insightful queries in a natural language interface.
+
+## Features
+
+- **Intuitive Chat Interface:** Interact with your data in a conversational manner using plain English queries.
+- **Streamlit UI:** Enjoy a visually appealing web app for seamless interaction with the chatbot.
+- **SQLite Support:** Utilize SQLite, a lightweight and embedded database engine, for efficient data storage.
+
+## Requirements
+
+- Python 3.x (https://www.python.org/downloads/)
+- OpenAI API Key (https://help.openai.com/en/articles/4936850-where-do-i-find-my-openai-api-key)
+- LangChain API Key ([invalid URL removed]) (Note: Check LangChain documentation for API key acquisition instructions)
+- `requirements.txt` (included)
+
+## Installation and Configuration
+
+```bash
+git clone https://github.com/paramrajyadav/sql_chat_bot.git
+```
+```bash
+cd sql-chatbot
+```
+```bash
+pip install -r requirements.txt
+```
+
+# Create a .env file and add these lines, replacing placeholders with your actual keys:
+OPENAI_API_KEY=your_openai_api_key
+
+LANGCHAIN_API_KEY=your_langchain_api_key
+
+## Usage
+Run the application: streamlit run app.py
+
+Access the Streamlit app in your web browser (usually http://localhost:8501).
+
+Interact with the chatbot by typing natural language queries in the input field.
+
+The chatbot will translate your queries into SQL and retrieve the desired information from your table.
+
+
+## Sample Queries
+"Show me all customers in California."
+
+"What are the average order values per product category?"
+
+"Find orders placed between January and March 2024."
+
+
+## Contributing
+
+We welcome contributions from the community! Here are some ways you can get involved:
+
+* **Bug reports:** If you find a bug, please create an issue on the GitHub repository. Be sure to include clear steps to reproduce the bug.
+* **Feature requests:** If you have an idea for a new feature, please create an issue on the GitHub repository. Describe the feature in detail and explain why it would be valuable.
+* **Pull requests:** If you have implemented a bug fix or feature, you can submit a pull request. Make sure to follow the contributing guidelines (if available) and test your changes thoroughly.

app.py

@@ -0,0 +1,77 @@
+import streamlit as st
+
+from aux_functions.aux_functions import *
+from aux_functions.add_data import *
+from aux_functions.create_tables import *
+from aux_functions.db_functions import *
+from aux_functions.chat_sql_function import *
+
+project_dir = os.getcwd()
+
+st.set_page_config(layout="wide")
+
+# Define the header of the app
+st.markdown("# SQL Chatbot")
+
+# Initialize session state if not already done
+if 'selected_index' not in st.session_state:
+    st.session_state['selected_index'] = 0
+
+def reset_app():
+    clean_data_folder(project_dir)
+    st.session_state['selected_index'] = 0
+
+
+selected_index = st.session_state['selected_index']
+
+if st.button("Reset the App", key="reset_app"):
+        reset_app()
+
+cols = st.columns(3)
+
+# Create menu buttons dynamically
+for i, label in enumerate(["Home", "Create Table", "Query"]):
+    if cols[i].button(label, key=f"menu_button_{i}"):
+        selected_index = i  # Select corresponding menu (Menu 1 = 1, Menu 2 = 2, etc.)
+        st.session_state['selected_index'] = selected_index
+
+
+
+
+
+if selected_index == 0:
+    st.markdown("# Upload your file")
+    handle_file_upload()
+
+elif selected_index == 1:
+    st.write("This is the display code for Create Table.")
+    built_table()
+    if st.button("Insert values in the database"):
+        file_name_from_file(project_dir)
+        drop_sqlite_database()
+        setup_database()
+        import_csv(project_dir)
+    st.markdown(" # Proceed to the QA Part")
+
+elif selected_index == 2:
+    st.write("# Enter your Question.")
+    query = st.text_input("Question")
+    if st.button("Initiate the Chatbot"):
+        x = initiate_chat(query)
+        st.write(x)
+
+button_col1, button_col2 = st.columns([1, 1])
+
+if button_col1.button("Go to previous Menu", key="prev_menu"):
+    if selected_index == 0:
+        st.write("Try Reset Button")
+    else:
+        selected_index -= 1
+        st.session_state['selected_index'] = selected_index
+
+if button_col2.button("Go to next Menu", key="next_menu"):
+    if selected_index == 2:
+        st.write("Try Reset Button")
+    else:
+        selected_index += 1
+        st.session_state['selected_index'] = selected_index

aux_data/column_names.txt

@@ -0,0 +1,10 @@
+experience_level
+employment_type
+job_title
+salary
+salary_currency
+salary_in_usd
+employee_residence
+remote_ratio
+company_location
+company_size

aux_data/file_name.txt

@@ -0,0 +1 @@
+salaries

aux_data/query.txt

@@ -0,0 +1,12 @@
+CREATE TABLE salaries (
+    experience_level VARCHAR(250),
+    employment_type VARCHAR(250),
+    job_title VARCHAR(250),
+    salary INT,
+    salary_currency VARCHAR(250),
+    salary_in_usd INT,
+    employee_residence VARCHAR(250),
+    remote_ratio INT,
+    company_location VARCHAR(250),
+    company_size VARCHAR(250)
+);

aux_functions/add_data.py

@@ -0,0 +1,79 @@
+import os
+import pandas as pd
+import streamlit as st
+import csv
+from aux_functions import *
+
+def handle_file_upload(upload_dir='data', aux_data_folder='aux_data'):
+    """
+    Handles the file upload, storage, and extraction of column names.
+    Stores the file name and column names in an auxiliary text file.
+    
+    Parameters:
+    upload_dir (str): The directory to store uploaded files.
+    aux_file (str): The name of the auxiliary text file to store file name and column names.
+    
+    Returns:
+    file_name (str): The name of the uploaded file.
+    column_names (list): The list of column names in the uploaded file.
+    """
+    # Create the directory if it doesn't exist
+    if not os.path.exists(upload_dir):
+        os.makedirs(upload_dir)
+
+    # File uploader
+    uploaded_file = st.file_uploader("Choose a file", type=['csv', 'xlsx'])
+
+    if uploaded_file is not None:
+        # Store file in the specified directory
+        # clean_data_folder()
+        file_name = uploaded_file.name
+        file_path = os.path.join(upload_dir, file_name)
+
+        for filename in os.listdir(upload_dir):
+            file_path_existing = os.path.join(upload_dir, filename)
+            if os.path.isfile(file_path_existing) and filename != file_name:
+                os.remove(file_path_existing)
+                # st.write(f"Deleted: {file_path_existing}")
+
+
+        
+        with open(file_path, "wb") as f:
+            f.write(uploaded_file.getbuffer())
+        
+        st.success(f"File {file_name} saved successfully!")
+        
+        # Read the file and get column names
+        if file_name.endswith('.csv'):
+            df = pd.read_csv(file_path)
+
+        else:
+            df = pd.read_xml(file_path)
+
+        column_names = df.columns.tolist()
+
+        
+        if not os.path.exists(aux_data_folder):
+            os.makedirs(aux_data_folder)
+
+# Define the filenames for the text files
+        file_name_filename = os.path.join(aux_data_folder, "file_name.txt")
+        column_names_filename = os.path.join(aux_data_folder, "column_names.txt")
+
+# Write file name to its respective text file
+        with open(file_name_filename, "w") as file_name_file:
+            file_name_without_extension = file_name.rsplit('.', 1)[0]
+            file_name_file.write(file_name_without_extension)
+
+# Write column names to its respective text file
+        with open(column_names_filename, "w") as column_names_file:
+            for col in column_names:
+                column_names_file.write(col + "\n")
+        st.markdown("## File upload Successful, Click on 'Go to next Menu' to proceed forward ")
+
+        return file_name, column_names
+    else:
+        st.info("Please upload a file")
+        return None, None
+    
+    

aux_functions/aux_functions.py

@@ -0,0 +1,38 @@
+import os
+import shutil
+
+def clean_data_folder(proj_dir=r"C:\Users\pcraj\OneDrive\Desktop\projects\new_test"):
+    try:
+        # Get the current working directory
+        
+
+
+        # Define the folder name to clean
+        folder_name = 'data'
+        folder_name2 = 'aux_data'
+
+        # Construct the full path to the folder
+        folder_path = os.path.join(proj_dir, folder_name)
+
+        # print(folder_name)
+
+        # Check if the folder exists
+        if os.path.exists(folder_path):
+            try:
+                # Delete the folder and its contents
+                shutil.rmtree(folder_path)
+                print(f"Successfully deleted the folder: {folder_path}")
+            except OSError as e:
+                print(f"Error: {folder_path} : {e.strerror}")
+            except PermissionError:
+                print(f"Permission error occurred while trying to delete contents of '{folder_name}'.")
+            except Exception as e:
+                print(f"An error occurred: {e}")
+        else:
+            print(f"The folder '{folder_path}' does not exist.")
+    
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+# Example usage:
+clean_data_folder()

aux_functions/chat_sql_function.py

@@ -0,0 +1,44 @@
+import os
+from dotenv import load_dotenv
+from langchain_community.utilities import SQLDatabase
+from langchain_community.agent_toolkits import create_sql_agent
+from langchain_openai import ChatOpenAI
+from huggingface_hub import notebook_login
+from huggingface_hub import hf_secrets
+
+
+# Load environment variables from .env file
+def initiate_chat(querry):
+    load_dotenv()
+
+    notebook_login()
+
+    # Set environment variables for API keys
+    # os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+    # os.environ["LANGCHAIN_API_KEY"] = os.getenv("LANGCHAIN_API_KEY")
+    # os.environ["LANGCHAIN_TRACING_V2"] = "true"
+
+    # Define the SQL database URI
+    db_uri = "sqlite:///db.db"
+
+    # Initialize the SQLDatabase object
+    db = SQLDatabase.from_uri(db_uri)
+    api_key = hf_secrets.get("open_api")
+
+    # Initialize the ChatOpenAI object with the desired model
+    llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, api_key=api_key)
+    # llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+
+    # Create the SQL agent
+    agent_executor = create_sql_agent(llm, db=db, agent_type="openai-tools", verbose=True)
+
+    # Define the query to get the sum of all salaries
+    
+
+    # Execute the query using the agent
+    try:
+        result = agent_executor.invoke(querry)
+        return result
+    except Exception as e:
+        print(f"Error: {e}")
+        return "Error: " + str(e)

aux_functions/create_tables.py

@@ -0,0 +1,69 @@
+import streamlit as st
+import os
+import pandas as pd
+
+def read_aux_data():
+    """
+    Reads the file name and column names from separate files within the aux_data directory.
+
+    Returns:
+    column_names (str): The column names read from column_names.txt.
+    file_name (str): The file name read from file_name.txt.
+
+    """
+    column_names_filename = 'aux_data/column_names.txt'
+    file_name_filename = 'aux_data/file_name.txt'
+    with open(column_names_filename, 'r') as column_file:
+        column_names = [line.strip() for line in column_file.readlines()]
+
+    # Read file name
+    with open(file_name_filename, 'r') as file_file:
+        file_name = file_file.read().strip()
+
+    return column_names, file_name
+
+def built_table():
+    st.title("File Upload, Data Preview, and SQL Schema Creation")
+
+    column_names, file_name = read_aux_data()
+
+    st.write("## Load and Display Saved Column Data")
+    st.write(column_names)
+
+    st.write("## Load and Display Sample Data")
+    file_path = os.path.join('data', file_name)
+
+    if os.path.exists(f"{file_path}.csv"):
+        df = pd.read_csv(f"{file_path}.csv")
+        st.write(df.head(20))
+    else:
+        st.write("Sample data not found.")
+
+    st.write("## Create SQL Schema")
+    sql_schema = {}
+    data_types = ['VARCHAR(250)', 'INT', 'FLOAT', 'DATE', 'DOUBLE']
+
+    for col in column_names:
+        selectbox_key = f"{col}_selectbox"
+        data_type = st.selectbox(f"Select data type for column '{col}'", [''] + data_types, key=selectbox_key)
+    
+        if data_type == '':
+            st.warning(f"Please select a data type for column '{col}'.")
+            st.stop()  # Stop execution if a data type is not selected
+        sql_schema[col] = data_type
+
+    st.write("### Generated SQL Schema")
+    create_table_query = f"CREATE TABLE {file_name} (\n"
+    create_table_query += ",\n".join([f"    {col} {dtype}" for col, dtype in sql_schema.items()])
+    create_table_query += "\n);"
+
+    st.code(create_table_query, language='sql')
+
+    if st.button("Schema is correct"):
+        # Store the query in a file
+        query_filename = 'aux_data/query.txt'
+        with open(query_filename, 'w') as query_file:
+            query_file.write(create_table_query)
+        st.success("Query has been saved.")
+
+    return None

aux_functions/db_functions.py

@@ -0,0 +1,123 @@
+import pandas as pd
+import csv
+import os
+def read_query_from_file(proj_dir=r"C:\Users\pcraj\OneDrive\Desktop\projects\new_test"):
+    file_path = 'aux_data/query.txt'
+    file_name=os.path.join(proj_dir,file_path)
+    try:
+        with open(file_name, 'r') as file:
+            CreateTable = file.read()
+            return CreateTable
+    except FileNotFoundError:
+        print(f"File {file_path} not found.")
+        return None
+    
+def file_name_from_file(proj_dir=r"C:\Users\pcraj\OneDrive\Desktop\projects\new_test"):
+    file_path = 'aux_data/file_name.txt'
+    file_name=os.path.join(proj_dir,file_path)
+    try:
+        with open(file_name, 'r') as file:
+            return file.read().strip()  # Remove leading/trailing whitespaces
+    except FileNotFoundError:
+        print(f"File {file_path} not found.")
+        return None
+    
+
+import sqlite3
+
+def drop_sqlite_database():
+    """
+    Deletes the SQLite database file at the specified path.
+
+    Parameters:
+    db_path (str): Path to the SQLite database file.
+
+    Returns:
+    None
+    """
+
+    db_path = 'db.db'
+    try:
+        if os.path.exists(db_path):
+            os.remove(db_path)
+            print(f"Database at {db_path} has been deleted.")
+        else:
+            print(f"No database file found at {db_path}.")
+    except Exception as e:
+        print(f"An error occurred while deleting the database: {e}")
+
+
+
+    
+
+
+def setup_database():
+
+    import sqlite3
+    create_table_query=read_query_from_file()
+    # create_table_query_2 =create_table_querry
+
+    # print(create_table_query_2)
+    connection = sqlite3.connect('db.db')
+    cursor = connection.cursor()
+
+    
+    cursor.execute(str(create_table_query))
+    connection.commit()
+    connection.close()
+   
+
+import sqlite3
+import csv
+
+def import_csv(proj_dir=r"C:\Users\pcraj\OneDrive\Desktop\projects\new_test"):
+    try:
+        connection = sqlite3.connect('db.db')
+        c = connection.cursor()
+        
+        file = file_name_from_file(proj_dir)  # Assuming this function is defined elsewhere
+        print("File to be imported:", file)
+        folder_name=os.path.join(proj_dir,"data")
+        
+        with open(f"{folder_name}/{file}.csv", 'r', encoding='utf-8') as f:
+            reader = csv.reader(f)
+            print("Opened CSV file")
+            next(reader)  # Skip the header row if present
+            
+            # Dynamically determine the number of columns in the CSV
+            first_row = next(reader)
+            num_columns = len(first_row)
+            placeholders = ','.join(['?'] * num_columns)
+            
+            # Re-insert the first row into the reader
+            reader = csv.reader(f)
+            next(reader)  # Skip the header row again
+            
+            for row in reader:
+                print("Inserting row:", row)
+                c.execute(f"INSERT INTO {file} VALUES ({placeholders})", row)
+            
+            connection.commit()
+            print("Data imported successfully")
+    except sqlite3.Error as e:
+        print("SQLite error:", e)
+    except Exception as e:
+        print("Error:", e)
+    finally:
+        if connection:
+            connection.close()
+            print("Database connection closed")
+
+
+    
+        
+    
+if __name__ == "__main__":
+    print("h1")
+    print(file_name_from_file())
+    print("h2")
+    drop_sqlite_database()
+    print("h3")
+    setup_database()
+    print("h4")
+    import_csv()

requirements.txt

@@ -0,0 +1,80 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+attrs==23.2.0
+blinker==1.8.2
+cachetools==5.3.3
+certifi==2024.6.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+dataclasses-json==0.6.7
+distro==1.9.0
+filelock==3.15.1
+frozenlist==1.4.1
+fsspec==2024.6.0
+gitdb==4.0.11
+GitPython==3.1.43
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.4
+idna==3.7
+Jinja2==3.1.4
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+langchain==0.2.5
+langchain-community==0.2.5
+langchain-core==0.2.7
+langchain-openai==0.1.8
+langchain-text-splitters==0.2.1
+langsmith==0.1.77
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+marshmallow==3.21.3
+mdurl==0.1.2
+multidict==6.0.5
+mypy-extensions==1.0.0
+numpy==1.26.4
+openai==1.34.0
+orjson==3.10.5
+packaging==24.1
+pandas==2.2.2
+pillow==10.3.0
+protobuf==4.25.3
+pyarrow==16.1.0
+pydantic==2.7.4
+pydantic_core==2.18.4
+pydeck==0.9.1
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.18.1
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+SQLAlchemy==2.0.30
+streamlit==1.35.0
+tenacity==8.3.0
+tiktoken==0.7.0
+toml==0.10.2
+toolz==0.12.1
+tornado==6.4.1
+tqdm==4.66.4
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.1
+watchdog==4.0.1
+yarl==1.9.4