successfully able to extract the description information back into a pandas dataframe. Need to look how to write to the database after every call instead of doing it in such large batches.

notebooks/google_job_rwtest.ipynb

@@ -83,17 +83,6 @@
     "#     return None"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# job_list= [\"Machine Learning Engineer\", \"Data Scientist\", \"Generative AI Engineer\", \"Solutions Engineer\", \"LLM Engineer\"]\n",
-    "# simple_city_state_list= [\"Menlo Park CA\", \"Palo Alto CA\", \"San Francisco CA\", \"Mountain View CA\"]\n",
-    "# sample = main(job_list, simple_city_state_list)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -109,6 +98,9 @@
     "import os\n",
     "from sqlalchemy import create_engine\n",
     "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()\n",
     "\n",
     "def google_job_search(job_title, city_state, start=0):\n",
     "    '''\n",
@@ -151,7 +143,7 @@
     "        return None\n",
     "\n",
     "def sql_dump(df, table):\n",
-    "    engine = create_engine(f\"postgresql://{os.getenv('MasterName')}:{os.getenv('MasterPass')}@{os.getenv('RDS_EndPoint')}:5432/postgres\")\n",
+    "    engine = create_engine(f\"postgresql://{os.getenv('PSQL_MASTER_NAME')}:{os.getenv('PSQL_KEY')}@{os.getenv('RDS_ENDPOINT')}:5432/postgres\")\n",
     "    with engine.connect() as conn:\n",
     "        df.to_sql(table, conn, if_exists='append', chunksize=20, method='multi', index=False)\n",
     "        print(f\"Dumped {df.shape} to SQL table {table}\")\n",
@@ -182,116 +174,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 17,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "City: Menlo Park CA Job: Data Engineer Start: 1\n",
-      "(10, 6)\n",
-      "City: San Francisco CA Job: Data Engineer Start: 0\n",
-      "(9, 6)\n",
-      "City: Mountain View CA Job: Data Analyst Start: 0\n",
-      "(10, 6)\n",
-      "City: Menlo Park CA Job: Data Analyst Start: 1\n",
-      "(10, 6)\n",
-      "City: Palo Alto CA Job: Data Analyst Start: 1\n",
-      "(10, 6)\n",
-      "City: San Francisco CA Job: Data Analyst Start: 0\n",
-      "(10, 6)\n",
-      "City: Menlo Park CA Job: Data Engineer Start: 0\n",
-      "(10, 6)\n",
-      "City: Palo Alto CA Job: Data Analyst Start: 0\n",
-      "(10, 6)\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTableDumped (9, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "City: Mountain View CA Job: Data Engineer Start: 1\n",
-      "(10, 6)\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "City: Mountain View CA Job: Big Data Engineer Start: 0\n",
-      "(10, 6)\n",
-      "City: San Francisco CA Job: Big Data Engineer Start: 1\n",
-      "(10, 6)\n",
-      "City: Menlo Park CA Job: Big Data Engineer Start: 1\n",
-      "(10, 6)\n",
-      "City: Menlo Park CA Job: Big Data Engineer Start: 0\n",
-      "(10, 6)\n",
-      "Dumped (10, 7) to SQL table leoTestTableDumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "City: Palo Alto CA Job: Data Engineer Start: 0\n",
-      "(10, 6)\n",
-      "Error occurred for search: Data Engineer in Palo Alto CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "Error occurred for search: Data Engineer in Mountain View CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "Error occurred for search: Data Analyst in Menlo Park CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "Error occurred for search: Data Analyst in Mountain View CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Error occurred for search: Data Engineer in San Francisco CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "City: Palo Alto CA Job: Big Data Engineer Start: 1\n",
-      "(10, 6)\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Error occurred for search: Big Data Engineer in Mountain View CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "Error occurred for search: Big Data Engineer in San Francisco CA\n",
-      "Error message: 'google_jobs_results'\n",
-      "City: San Francisco CA Job: Data Analyst Start: 1\n",
-      "(10, 6)\n",
-      "Dumped (10, 7) to SQL table leoTestTable\n",
-      "Rows affected: None\n",
-      "Error occurred for search: Big Data Engineer in Palo Alto CA\n",
-      "Error message: 'google_jobs_results'\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "job_list = [\"Data Analyst\", \"Data Engineer\", \"Big Data Engineer\"]\n",
-    "simple_city_state_list = [\"Menlo Park CA\", \"Palo Alto CA\", \"San Francisco CA\", \"Mountain View CA\"]\n",
-    "main(job_list, simple_city_state_list)"
+    "# job_list = [\"Data Analyst\", \"Data Engineer\", \"Big Data Engineer\"]\n",
+    "# simple_city_state_list = [\"Menlo Park CA\", \"Palo Alto CA\", \"San Francisco CA\", \"Mountain View CA\"]\n",
+    "# main(job_list, simple_city_state_list)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -301,7 +192,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -309,7 +200,7 @@
     "from sqlalchemy import create_engine\n",
     "\n",
     "def read_data_from_db(table_name):\n",
-    "    engine = create_engine(f\"postgresql://{os.getenv('MasterName')}:{os.getenv('MasterPass')}@{os.getenv('RDS_EndPoint')}:5432/postgres\")\n",
+    "    engine = create_engine(f\"postgresql://{os.getenv('PSQL_MASTER_NAME')}:{os.getenv('PSQL_KEY')}@{os.getenv('RDS_ENDPOINT')}:5432/postgres\")\n",
     "    \n",
     "    try:\n",
     "        with engine.connect() as conn:\n",
@@ -325,7 +216,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -334,7 +225,7 @@
        "(417, 7)"
       ]
      },
-     "execution_count": 32,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -345,7 +236,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -563,7 +454,7 @@
        "[417 rows x 7 columns]"
       ]
      },
-     "execution_count": 33,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -574,17 +465,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
     "# get the list of unique title, company_name pairs\n",
-    "title_company = data24_df[['title', 'company_name', 'description']].drop_duplicates()"
+    "title_company = data24_df[['title', 'company_name', 'location', 'description']].drop_duplicates()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -612,9 +503,6 @@
        "      <th>company_name</th>\n",
        "      <th>location</th>\n",
        "      <th>description</th>\n",
-       "      <th>extensions</th>\n",
-       "      <th>job_id</th>\n",
-       "      <th>retrieve_date</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -624,9 +512,6 @@
        "      <td>Nuvolum</td>\n",
        "      <td>San Francisco, CA</td>\n",
        "      <td>Nuvolum combines innovative, data-driven strat...</td>\n",
-       "      <td>{\"3 days ago\",Full-time,\"No degree mentioned\"}</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
@@ -634,9 +519,6 @@
        "      <td>Sunrun</td>\n",
        "      <td>San Francisco, CA (+1 other)</td>\n",
        "      <td>Everything we do at Sunrun is driven by a dete...</td>\n",
-       "      <td>{\"12 days ago\",Full-time,\"Health insurance\",\"D...</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJTci4gU3RyYXRlZ3kgYW5kIEJ1c2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
@@ -644,9 +526,6 @@
        "      <td>Side</td>\n",
        "      <td>Anywhere</td>\n",
        "      <td>Side, Inc. seeks Business Intelligence Analyst...</td>\n",
-       "      <td>{\"11 days ago\",\"151,736–157,000 a year\",\"Work ...</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
@@ -654,9 +533,6 @@
        "      <td>TekNavigators Staffing</td>\n",
        "      <td>San Francisco, CA</td>\n",
        "      <td>Role: Senior BI Developer\\n\\nLocation: San Fra...</td>\n",
-       "      <td>{\"20 hours ago\",Contractor,\"No degree mentioned\"}</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJTZW5pb3IgQnVzaW5lc3MgSW50ZW...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
@@ -664,9 +540,6 @@
        "      <td>FIS Fidelity National Information Services</td>\n",
        "      <td>San Francisco, CA</td>\n",
        "      <td>Position Type : Full time Type Of Hire : Exper...</td>\n",
-       "      <td>{\"19 days ago\",Full-time}</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJTZW5pb3IgQnVzaW5lc3MgSW50ZW...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -674,9 +547,6 @@
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>412</th>\n",
@@ -684,9 +554,6 @@
        "      <td>Medtronic</td>\n",
        "      <td>Anywhere</td>\n",
        "      <td>Careers that Change Lives\\n\\nWe are looking fo...</td>\n",
-       "      <td>{\"10 days ago\",\"Work from home\",Full-time,\"No ...</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>413</th>\n",
@@ -694,9 +561,6 @@
        "      <td>Keck Medicine of USC</td>\n",
        "      <td>Alhambra, CA</td>\n",
        "      <td>Actively design and develop ETL solutions that...</td>\n",
-       "      <td>{\"13 days ago\",Full-time}</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJJVCBBbmFseXN0LCBCdXNpbmVzcy...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>414</th>\n",
@@ -704,9 +568,6 @@
        "      <td>Deutsch LA</td>\n",
        "      <td>Los Angeles, CA</td>\n",
        "      <td>DIRECTOR, BUSINESS INTELLIGENCE\\n\\nWe are seek...</td>\n",
-       "      <td>{\"3 days ago\",Full-time,\"No degree mentioned\"}</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJEaXJlY3RvciwgQnVzaW5lc3MgSW...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>415</th>\n",
@@ -714,9 +575,6 @@
        "      <td>U.S. Bank</td>\n",
        "      <td>Los Angeles, CA</td>\n",
        "      <td>At U.S. Bank, we’re on a journey to do our bes...</td>\n",
-       "      <td>{\"3 days ago\",Full-time,\"Health insurance\",\"De...</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>416</th>\n",
@@ -724,13 +582,10 @@
        "      <td>BIGO</td>\n",
        "      <td>Los Angeles, CA</td>\n",
        "      <td>Location: 10250 Constellation Blvd., Century C...</td>\n",
-       "      <td>{\"1 day ago\",Full-time,\"Health insurance\",\"Den...</td>\n",
-       "      <td>eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...</td>\n",
-       "      <td>2024-05-04</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>417 rows × 7 columns</p>\n",
+       "<p>405 rows × 4 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -760,55 +615,38 @@
        "415                                   U.S. Bank               Los Angeles, CA   \n",
        "416                                        BIGO               Los Angeles, CA   \n",
        "\n",
-       "                                           description  \\\n",
-       "0    Nuvolum combines innovative, data-driven strat...   \n",
-       "1    Everything we do at Sunrun is driven by a dete...   \n",
-       "2    Side, Inc. seeks Business Intelligence Analyst...   \n",
-       "3    Role: Senior BI Developer\\n\\nLocation: San Fra...   \n",
-       "4    Position Type : Full time Type Of Hire : Exper...   \n",
-       "..                                                 ...   \n",
-       "412  Careers that Change Lives\\n\\nWe are looking fo...   \n",
-       "413  Actively design and develop ETL solutions that...   \n",
-       "414  DIRECTOR, BUSINESS INTELLIGENCE\\n\\nWe are seek...   \n",
-       "415  At U.S. Bank, we’re on a journey to do our bes...   \n",
-       "416  Location: 10250 Constellation Blvd., Century C...   \n",
+       "                                           description  \n",
+       "0    Nuvolum combines innovative, data-driven strat...  \n",
+       "1    Everything we do at Sunrun is driven by a dete...  \n",
+       "2    Side, Inc. seeks Business Intelligence Analyst...  \n",
+       "3    Role: Senior BI Developer\\n\\nLocation: San Fra...  \n",
+       "4    Position Type : Full time Type Of Hire : Exper...  \n",
+       "..                                                 ...  \n",
+       "412  Careers that Change Lives\\n\\nWe are looking fo...  \n",
+       "413  Actively design and develop ETL solutions that...  \n",
+       "414  DIRECTOR, BUSINESS INTELLIGENCE\\n\\nWe are seek...  \n",
+       "415  At U.S. Bank, we’re on a journey to do our bes...  \n",
+       "416  Location: 10250 Constellation Blvd., Century C...  \n",
        "\n",
-       "                                            extensions  \\\n",
-       "0       {\"3 days ago\",Full-time,\"No degree mentioned\"}   \n",
-       "1    {\"12 days ago\",Full-time,\"Health insurance\",\"D...   \n",
-       "2    {\"11 days ago\",\"151,736–157,000 a year\",\"Work ...   \n",
-       "3    {\"20 hours ago\",Contractor,\"No degree mentioned\"}   \n",
-       "4                            {\"19 days ago\",Full-time}   \n",
-       "..                                                 ...   \n",
-       "412  {\"10 days ago\",\"Work from home\",Full-time,\"No ...   \n",
-       "413                          {\"13 days ago\",Full-time}   \n",
-       "414     {\"3 days ago\",Full-time,\"No degree mentioned\"}   \n",
-       "415  {\"3 days ago\",Full-time,\"Health insurance\",\"De...   \n",
-       "416  {\"1 day ago\",Full-time,\"Health insurance\",\"Den...   \n",
-       "\n",
-       "                                                job_id retrieve_date  \n",
-       "0    eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...    2024-05-04  \n",
-       "1    eyJqb2JfdGl0bGUiOiJTci4gU3RyYXRlZ3kgYW5kIEJ1c2...    2024-05-04  \n",
-       "2    eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...    2024-05-04  \n",
-       "3    eyJqb2JfdGl0bGUiOiJTZW5pb3IgQnVzaW5lc3MgSW50ZW...    2024-05-04  \n",
-       "4    eyJqb2JfdGl0bGUiOiJTZW5pb3IgQnVzaW5lc3MgSW50ZW...    2024-05-04  \n",
-       "..                                                 ...           ...  \n",
-       "412  eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...    2024-05-04  \n",
-       "413  eyJqb2JfdGl0bGUiOiJJVCBBbmFseXN0LCBCdXNpbmVzcy...    2024-05-04  \n",
-       "414  eyJqb2JfdGl0bGUiOiJEaXJlY3RvciwgQnVzaW5lc3MgSW...    2024-05-04  \n",
-       "415  eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...    2024-05-04  \n",
-       "416  eyJqb2JfdGl0bGUiOiJCdXNpbmVzcyBJbnRlbGxpZ2VuY2...    2024-05-04  \n",
-       "\n",
-       "[417 rows x 7 columns]"
+       "[405 rows x 4 columns]"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "data24_df"
+    "title_company"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data24_df.to_csv('data24.csv', index=False)"
    ]
   },
   {