Delete pages/test.py

pages/test.py

@@ -1,236 +0,0 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-from scipy.stats import pearsonr
-from scipy.spatial.distance import euclidean
-from sentence_transformers import SentenceTransformer
-import groq
-import math
-import json
-from huggingface_hub import HfApi
-import io
-from pdfminer.high_level import extract_text
-import pyarrow.feather as feather
-import re
-from datetime import datetime, timedelta
-
-HF_TOKEN = st.secrets["HF_TOKEN"]
-HF_USERNAME = st.secrets["HF_USERNAME"]
-DATASET_NAME = "jobeasz"
-
-@st.cache_data(ttl=3600)
-def load_and_concat_data():
-    api = HfApi()
-    dataset_files = api.list_repo_files(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", repo_type="dataset")
-    feather_files = [file for file in dataset_files if file.endswith('.feather')]
-
-    all_data = []
-    for file in feather_files:
-        try:
-            file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
-            df = feather.read_feather(file_content)
-            all_data.append(df)
-        except Exception:
-            pass  # Silently skip files that can't be processed
-
-    if not all_data:
-        return pd.DataFrame()
-
-    concatenated_df = pd.concat(all_data, ignore_index=True)
-    
-    columns_to_keep = [
-        'site', 'job_url', 'title', 'company', 'location',
-        'job_type', 'date_posted', 'is_remote', 'company_url', 'description'
-    ]
-    filtered_df = concatenated_df[columns_to_keep].reset_index(drop=True)
-    filtered_df['date_posted'] = pd.to_datetime(filtered_df['date_posted'], errors='coerce')
-    
-    # Drop duplicates and rows with NaT in date_posted
-    filtered_df = filtered_df.drop_duplicates().dropna(subset=['date_posted'])
-    #filtering based on data in 2024
-    filtered_df = filtered_df[filtered_df['date_posted'].dt.year==2024]
-    # Convert titles and company name to lowercase
-    filtered_df['title'] = filtered_df['title'].str.lower()
-    filtered_df['company'] = filtered_df['company'].str.lower()
-    
-    # Function to clean the location
-    def clean_location(location):
-        if pd.isna(location):
-            return location  # Return NaN as is
-        # Convert to lowercase
-        location = location.lower()
-        # Remove ', us' or ', usa' from the end using regex
-        location = re.sub(r',\s*(us|usa)$', '', location)
-        return location
-
-    # Clean the location in place
-    filtered_df['location'] = filtered_df['location'].apply(clean_location)
-    #added new line to drop duplicate records
-    filtered_df = filtered_df.drop_duplicates()
-    
-    return filtered_df
-
-def remove_special_chars(text):
-    if pd.isna(text):
-        return text
-    # Remove special characters and markdown formatting
-    cleaned_text = re.sub(r'[*\n\-_]', ' ', text)
-    # Remove extra whitespace
-    cleaned_text = ' '.join(cleaned_text.split())
-    return cleaned_text
-
-@st.cache_resource
-def load_models():
-    return {
-        'minilm': SentenceTransformer('all-MiniLM-L6-v2'),
-        'mpnet': SentenceTransformer('all-mpnet-base-v2'),
-        'paraphrase': SentenceTransformer('paraphrase-MiniLM-L6-v2')
-    }
-
-def generate_embeddings(text, models):
-    return {
-        'minilm': models['minilm'].encode(text),
-        'mpnet': models['mpnet'].encode(text),
-        'paraphrase': models['paraphrase'].encode(text)
-    }
-
-def calculate_similarities(job_embeddings, resume_embedding):
-    similarities = []
-    for job_embedding in job_embeddings:
-        job_emb = np.array(job_embedding).reshape(1, -1)
-        res_emb = resume_embedding.reshape(1, -1)
-        cosine_sim = cosine_similarity(job_emb, res_emb)[0][0]
-        pearson_corr = pearsonr(job_embedding, resume_embedding)[0]
-        euclidean_dist = euclidean(job_embedding, resume_embedding)
-        similarities.append({
-            'cosine': cosine_sim,
-            'pearson': pearson_corr,
-            'euclidean': euclidean_dist
-        })
-    return similarities
-
-def get_top_matches(df, n=50):
-    top_matches = pd.DataFrame()
-    for model_name in ['minilm', 'mpnet', 'paraphrase']:
-        for metric in ['cosine', 'pearson', 'euclidean']:
-            col_name = f'{model_name}_{metric}'
-            ascending = metric == 'euclidean'
-            top_n = df.nsmallest(n, col_name) if ascending else df.nlargest(n, col_name)
-            top_n['model'] = model_name
-            top_n['metric'] = metric
-            top_matches = pd.concat([top_matches, top_n])
-    return top_matches.drop_duplicates().head(150)
-
-@st.cache_data
-def evaluate_with_groq(resume_text, job_description_text, client):
-    prompt = f"""
-    Resume: {resume_text}
-    Job Description: {job_description_text}
-    Based on the above information, rate the match quality on a scale of 0-100 and provide reasoning.
-    Return your response in the following JSON format:
-    {{ "score": <integer between 0 and 100>, "reasoning": "<your explanation>" }}
-    """
-    response = client.chat.completions.create(
-        messages=[
-            {"role": "user", "content": prompt}
-        ],
-        model="mixtral-8x7b-32768",
-        max_tokens=200,
-    )
-    return json.loads(response.choices[0].message.content)
-
-def display_data_explorer(df):
-    st.subheader("Data Explorer")
-    items_per_page = 15
-    num_pages = math.ceil(len(df) / items_per_page)
-    col1, col2, col3 = st.columns([1, 3, 1])
-    with col2:
-        page = st.number_input("Page", min_value=1, max_value=num_pages, value=1)
-    start_idx = (page - 1) * items_per_page
-    end_idx = start_idx + items_per_page
-    page_df = df.iloc[start_idx:end_idx]
-    
-    def make_clickable(url, text):
-        return f'<a href="{url}" target="_blank" style="color: #4e79a7;">{text}</a>'
-    
-    page_df['job_url'] = page_df.apply(lambda row: make_clickable(row['job_url'], 'Link'), axis=1)
-    page_df['company_url'] = page_df.apply(lambda row: make_clickable(row['company_url'], row['company']), axis=1)
-    
-    display_columns = ['title', 'company_url', 'location', 'job_type', 'date_posted', 'job_url', 'groq_score', 'groq_reasoning']
-    st.write(page_df[display_columns].to_html(escape=False, index=False), unsafe_allow_html=True)
-    
-    col1, col2, col3 = st.columns([1, 3, 1])
-    with col2:
-        st.write(f"Page {page} of {num_pages}")
-
-def read_file_content(uploaded_file):
-    if uploaded_file.type == "application/pdf":
-        pdf_reader = io.BytesIO(uploaded_file.getvalue())
-        return extract_text(pdf_reader)
-    else:
-        return uploaded_file.getvalue().decode("utf-8", errors="ignore")
-
-def main():
-    st.title("Resume-Job Matcher")
-
-    # Load data
-    df = load_and_concat_data()
-
-    # Filter data for the latest 3 days
-    current_date = datetime.now().date()
-    date_3_days_ago = current_date - timedelta(days=3)
-    df['date'] = df['date_posted'].dt.date
-    df_filtered = df[df['date'] >= date_3_days_ago]
-
-    # Print count of records for each day
-    for date in [current_date, current_date - timedelta(days=1), current_date - timedelta(days=2)]:
-        count = df_filtered[df_filtered['date'] == date].shape[0]
-        st.write(f"Records for {date}: {count}")
-
-    # Clean description and create embeddings
-    models = load_models()
-    df_filtered['cleaned_description'] = df_filtered['description'].apply(remove_special_chars)
-    
-    for model_name in ['minilm', 'mpnet', 'paraphrase']:
-        df_filtered[f'embeddings_{model_name}'] = df_filtered['cleaned_description'].apply(lambda x: models[model_name].encode(x))
-
-    uploaded_file = st.file_uploader("Upload your resume", type=["txt", "pdf"], key="resume_uploader")
-    if uploaded_file is not None:
-        try:
-            resume_text = read_file_content(uploaded_file)
-            cleaned_resume = remove_special_chars(resume_text)
-            st.subheader("Parsed Resume")
-            st.text(cleaned_resume)
-
-            resume_embeddings = generate_embeddings(cleaned_resume, models)
-
-            for model_name in ['minilm', 'mpnet', 'paraphrase']:
-                similarities = calculate_similarities(df_filtered[f'embeddings_{model_name}'].tolist(), resume_embeddings[model_name])
-                for metric in ['cosine', 'pearson', 'euclidean']:
-                    df_filtered[f'{model_name}_{metric}'] = [s[metric] for s in similarities]
-
-            top_matches = get_top_matches(df_filtered, 50)
-            st.subheader("Top 150 Matches (Before Groq Evaluation)")
-            st.dataframe(top_matches[['title', 'company', 'location', 'model', 'metric']])
-
-            groq_api_key = st.text_input("Enter your Groq API Key", type="password")
-            if groq_api_key:
-                client = groq.Groq(api_key=groq_api_key)
-                st.subheader("Evaluating matches with Groq...")
-                progress_bar = st.progress(0)
-                for i, row in enumerate(top_matches.itertuples()):
-                    groq_result = evaluate_with_groq(cleaned_resume, row.description, client)
-                    top_matches.at[row.Index, 'groq_score'] = groq_result['score']
-                    top_matches.at[row.Index, 'groq_reasoning'] = groq_result['reasoning']
-                    progress_bar.progress((i + 1) / len(top_matches))
-
-                top_100_matches = top_matches.nlargest(100, 'groq_score')
-                st.subheader("Top 100 Matches After Groq Evaluation")
-                display_data_explorer(top_100_matches)
-
-        except Exception as e:
-            st.error(f"An error occurred while processing the file: {str(e)}")
-
-if __name__ == "__main__":
-    main()