Update app.py

app.py

@@ -55,8 +55,6 @@ HF_TOKEN = st.secrets["HF_TOKEN"]
 HF_USERNAME = st.secrets["HF_USERNAME"]
 DATASET_NAME = "jobeasz"
 
-
-
 @st.cache_data(ttl=3600)
 def load_and_concat_data():
     api = HfApi()
@@ -68,64 +66,56 @@ def load_and_concat_data():
         try:
             file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
             
-            # Use PyArrow to read CSV
-            read_options = csv.ReadOptions(column_names=[
-                'site', 'job_url', 'title', 'company', 'location',
-                'job_type', 'date_posted', 'is_remote', 'company_url'
-            ])
-            parse_options = csv.ParseOptions(delimiter=',')
-            convert_options = csv.ConvertOptions(
-                timestamp_parsers=['%Y-%m-%d']
-            )
+            # Use CSV sniffer to detect delimiter and number of columns
+            with open(file_content, 'r') as f:
+                sample = f.read(1024)
+                sniffer = csv.Sniffer()
+                dialect = sniffer.sniff(sample)
+                f.seek(0)
+                
+                # Use pyarrow for more flexible parsing
+                parse_options = csv.ParseOptions(delimiter=dialect.delimiter)
+                table = csv.read_csv(file_content, parse_options=parse_options)
+                
+                # Convert to pandas DataFrame
+                df = table.to_pandas()
+            
+            # Ensure all required columns are present, fill with NaN if missing
+            required_columns = ['site', 'job_url', 'title', 'company', 'location', 'job_type', 'date_posted', 'is_remote', 'company_url']
+            for col in required_columns:
+                if col not in df.columns:
+                    df[col] = pd.NA
             
-            table = csv.read_csv(file_content, read_options=read_options, parse_options=parse_options, convert_options=convert_options)
-            all_data.append(table)
+            # Select only the required columns
+            df = df[required_columns]
+            
+            all_data.append(df)
         except Exception as e:
             print(f"Error processing file {file}: {str(e)}")
+            continue
 
     if not all_data:
-        return pa.Table.from_pandas(pd.DataFrame())
-
-    # Concatenate all tables
-    concatenated_table = pa.concat_tables(all_data)
-
-    # Filter for 2024 data
-    mask = pc.year(concatenated_table['date_posted']) == 2024
-    filtered_table = concatenated_table.filter(mask)
-
-    # Convert titles and company names to lowercase
-    filtered_table = filtered_table.set_column(
-        filtered_table.schema.get_field_index('title'),
-        'title',
-        pc.utf8_lower(filtered_table['title'])
-    )
-    filtered_table = filtered_table.set_column(
-        filtered_table.schema.get_field_index('company'),
-        'company',
-        pc.utf8_lower(filtered_table['company'])
-    )
+        return pd.DataFrame()
 
-    # Clean location
+    concatenated_df = pd.concat(all_data, ignore_index=True)
+    
+    # Perform data cleaning and processing
+    concatenated_df['date_posted'] = pd.to_datetime(concatenated_df['date_posted'], errors='coerce')
+    concatenated_df = concatenated_df.dropna(subset=['date_posted'])
+    concatenated_df = concatenated_df[concatenated_df['date_posted'].dt.year == 2024]
+    concatenated_df['title'] = concatenated_df['title'].str.lower()
+    concatenated_df['company'] = concatenated_df['company'].str.lower()
+    
     def clean_location(location):
-        if location is None:
-            return None
-        location = location.lower()
+        if pd.isna(location):
+            return location
+        location = str(location).lower()
         return re.sub(r',\s*(us|usa)$', '', location)
 
-    cleaned_locations = pc.map(filtered_table['location'], clean_location)
-    filtered_table = filtered_table.set_column(
-        filtered_table.schema.get_field_index('location'),
-        'location',
-        cleaned_locations
-    )
-
-    # Remove duplicates
-    filtered_table = filtered_table.group_by(filtered_table.column_names).aggregate([])
-
-    # Convert to pandas DataFrame for compatibility with the rest of your code
-    filtered_df = filtered_table.to_pandas()
-
-    return filtered_df
+    concatenated_df['location'] = concatenated_df['location'].apply(clean_location)
+    concatenated_df = concatenated_df.drop_duplicates()
+    
+    return concatenated_df
     
 @st.cache_data()
 def get_unique_values(df):