Upload app.py

app.py

@@ -207,7 +207,7 @@ class linkedin_scraper:
         return scrap_job_title if len(matched_words) > 1 else np.nan
 
 
-    def scrap_company_data(driver, job_title_input, job_count):
+    def scrap_company_data(driver, job_title_input):
 
         # scraping the Company Data
         company = driver.find_elements(by=By.CSS_SELECTOR, value='h4[class="base-search-card__subtitle"]')
@@ -235,13 +235,10 @@ class linkedin_scraper:
         df = df.dropna()
         df.reset_index(drop=True, inplace=True)
 
-        # Filter Job Title Based on User Input
-        df = df.iloc[:job_count, :]
-
         return df 
         
 
-    def scrap_job_description(driver, df):
+    def scrap_job_description(driver, df, job_count):
         
         # Get URL into List
         website_url = df['Website URL'].tolist()
@@ -249,47 +246,47 @@ class linkedin_scraper:
         # Scrap the Job Description
         job_description = []
         for i in range(0, len(website_url)):
-            # Open the URL
-            driver.get(website_url[i])
-            driver.implicitly_wait(10)
-            time.sleep(1)
-
             try:
-                # Click on Show More Button
-                driver.find_element(by=By.CSS_SELECTOR, value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
+                # Open the URL
+                driver.get(website_url[i])
                 driver.implicitly_wait(10)
                 time.sleep(1)
-
+            
                 # Click on Show More Button
                 driver.find_element(by=By.CSS_SELECTOR, value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
                 driver.implicitly_wait(10)
                 time.sleep(1)
 
-            except NoSuchElementException:
-                # Open the URL
-                driver.get('https://www.google.com/')
-                driver.get(website_url[i])
-                driver.implicitly_wait(10)
-                time.sleep(1)
-
                 # Click on Show More Button
                 driver.find_element(by=By.CSS_SELECTOR, value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
                 driver.implicitly_wait(10)
                 time.sleep(1)
 
-            # Get Job Description
-            description = driver.find_elements(by=By.CSS_SELECTOR, value='div[class="show-more-less-html__markup relative overflow-hidden"]')
-            driver.implicitly_wait(10)
-            data = [i.text for i in description][0]
+                # Get Job Description
+                description = driver.find_elements(by=By.CSS_SELECTOR, value='div[class="show-more-less-html__markup relative overflow-hidden"]')
+                driver.implicitly_wait(10)
+                data = [i.text for i in description][0]
 
-            if len(data.strip()) > 0:
-                job_description.append(data)
-            else:
+                if len(data.strip()) > 0:
+                    job_description.append(data)
+                else:
+                    job_description.append('Description Not Available')
+                
+                # Check Description Count Meets User Job Count
+                if len([i for i in job_description if i != 'Description Not Available']) >= job_count:
+                    break
+            
+            # If URL cannot Loading Properly 
+            except:
                 job_description.append('Description Not Available')
 
+        
+        # Filter the Job Description
+        df = df.iloc[:len(job_description), :]
 
         # Add Job Description in Dataframe
         df['Job Description'] = pd.DataFrame(job_description, columns=['Description'])
+        df['Job Description'] = df['Job Description'].apply(lambda x: np.nan if x=='Description Not Available' else x)
         df = df.dropna()
         df.reset_index(drop=True, inplace=True)
         return df
@@ -335,10 +332,10 @@ class linkedin_scraper:
                         linkedin_scraper.link_open_scrolldown(driver, link, job_count)
 
                     with st.spinner('scraping Company Data...'):
-                        df = linkedin_scraper.scrap_company_data(driver, job_title_input, job_count)
+                        df = linkedin_scraper.scrap_company_data(driver, job_title_input)
                     
                     with st.spinner('Scraping Job Description Data...'):
-                        df_final = linkedin_scraper. scrap_job_description(driver, df)
+                        df_final = linkedin_scraper. scrap_job_description(driver, df, job_count)
                     
                     # Display the Data in User Interface
                     linkedin_scraper.display_data_userinterface(df_final)