Update app.py
Browse files
app.py
CHANGED
@@ -19,26 +19,20 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
19 |
|
20 |
def scrape_upwork_data(search_query, num_jobs, page):
|
21 |
# Setup Chrome options for remote WebDriver
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
chrome_options.add_argument("--disable-extensions")
|
27 |
-
chrome_options.add_argument("--no-sandbox")
|
28 |
-
chrome_options.add_argument("--disable-dev-shm-usage")
|
29 |
|
30 |
# Setup the remote WebDriver
|
31 |
-
driver
|
32 |
-
command_executor='http://localhost:4444/wd/hub',
|
33 |
-
options=chrome_options
|
34 |
-
)
|
35 |
-
|
36 |
job_listings = []
|
37 |
try:
|
38 |
url = f'https://www.upwork.com/nx/search/jobs?amount=500-&hourly_rate=25-&location=Americas,Europe,Australia%20and%20New%20Zealand,Canada,India,Switzerland,United%20States&per_page=50&q={search_query}&sort=recency&t=0,1&page={page}'
|
39 |
driver.get(url)
|
40 |
-
|
41 |
time.sleep(5) # Wait for the page to load
|
|
|
42 |
jobs = driver.find_elements(By.CSS_SELECTOR, 'article[data-test="JobTile"]')
|
43 |
|
44 |
for job in jobs:
|
|
|
19 |
|
20 |
def scrape_upwork_data(search_query, num_jobs, page):
|
21 |
# Setup Chrome options for remote WebDriver
|
22 |
+
options = webdriver.ChromeOptions()
|
23 |
+
options.add_argument('--headless')
|
24 |
+
options.add_argument('--no-sandbox')
|
25 |
+
options.add_argument('--disable-dev-shm-usage')
|
|
|
|
|
|
|
26 |
|
27 |
# Setup the remote WebDriver
|
28 |
+
driver=webdriver.Chrome(options=options)
|
|
|
|
|
|
|
|
|
29 |
job_listings = []
|
30 |
try:
|
31 |
url = f'https://www.upwork.com/nx/search/jobs?amount=500-&hourly_rate=25-&location=Americas,Europe,Australia%20and%20New%20Zealand,Canada,India,Switzerland,United%20States&per_page=50&q={search_query}&sort=recency&t=0,1&page={page}'
|
32 |
driver.get(url)
|
33 |
+
driver.set_window_size(1080, 720)
|
34 |
time.sleep(5) # Wait for the page to load
|
35 |
+
page_title = driver.title
|
36 |
jobs = driver.find_elements(By.CSS_SELECTOR, 'article[data-test="JobTile"]')
|
37 |
|
38 |
for job in jobs:
|