Update app.py

app.py

@@ -65,47 +65,46 @@ def load_and_concat_data():
     for file in csv_files:
         try:
             file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
-            
-            # Use PyArrow's CSV reading capabilities
-            read_options = csv.ReadOptions(use_threads=True)
-            parse_options = csv.ParseOptions(delimiter=',')  # Adjust delimiter if needed
-            convert_options = csv.ConvertOptions(
-                column_types={
-                    'date_posted': pa.timestamp('s'),
-                    'is_remote': pa.bool_()
-                },
-                strings_can_be_null=True
-            )
-            
-            table = csv.read_csv(file_content, read_options=read_options, parse_options=parse_options, convert_options=convert_options)
-            df = table.to_pandas()
-            
-            # Perform data cleaning and processing
-            df['date_posted'] = pd.to_datetime(df['date_posted'], errors='coerce')
-            df = df.dropna(subset=['date_posted'])
-            df = df[df['date_posted'].dt.year == 2024]
-            df['title'] = df['title'].str.lower()
-            df['company'] = df['company'].str.lower()
-            
-            def clean_location(location):
-                if pd.isna(location):
-                    return location
-                location = str(location).lower()
-                return re.sub(r',\s*(us|usa)$', '', location)
-            
-            df['location'] = df['location'].apply(clean_location)
-            
+            df = pd.read_csv(file_content, engine='pyarrow')
             all_data.append(df)
-        except Exception as e:
-            print(f"Error processing file {file}: {str(e)}")
-            continue
+        except Exception:
+            pass  # Silently skip files that can't be processed
 
     if not all_data:
         return pd.DataFrame()
 
     concatenated_df = pd.concat(all_data, ignore_index=True)
-    filtered_df = concatenated_df.drop_duplicates().reset_index(drop=True)
-
+    
+    columns_to_keep = [
+        'site', 'job_url', 'title', 'company', 'location',
+        'job_type', 'date_posted', 'is_remote', 'company_url'
+    ]
+    filtered_df = concatenated_df[columns_to_keep].reset_index(drop=True)
+    filtered_df['date_posted'] = pd.to_datetime(filtered_df['date_posted'], errors='coerce')
+    
+    # Drop duplicates and rows with NaT in date_posted
+    filtered_df = filtered_df.drop_duplicates().dropna(subset=['date_posted'])
+    #filtering based on data in 2024
+    filtered_df = filtered_df[filtered_df['date_posted'].dt.year==2024]
+    # Convert titles and company name to lowercase
+    filtered_df['title'] = filtered_df['title'].str.lower()
+    filtered_df['company'] = filtered_df['company'].str.lower()
+    
+    # Function to clean the location
+    def clean_location(location):
+        if pd.isna(location):
+            return location  # Return NaN as is
+        # Convert to lowercase
+        location = location.lower()
+        # Remove ', us' or ', usa' from the end using regex
+        location = re.sub(r',\s*(us|usa)$', '', location)
+        return location
+
+    # Clean the location in place
+    filtered_df['location'] = filtered_df['location'].apply(clean_location)
+    #added new line to drop duplciate records
+    filtered_df = filtered_df.drop_duplicates()
+    
     return filtered_df
     
 @st.cache_data()