able to parse objects out of the job description, but extracting in mass w/ error handling is a little difficult. Most recent run is in parse_description_test notebook thats able to connect to psql and then extract a sample set of 74 descriptions.

utils/google_jobs.py

@@ -14,11 +14,10 @@ def google_job_search(job_title, city_state, start=0):
     '''
     job_title(str): "Data Scientist", "Data Analyst"
     city_state(str): "Denver, CO"
-    post_age,(str)(optional): "3day", "week", "month"
     '''
     query = f"{job_title} {city_state}"
     params = {
-        "api_key": os.getenv('SerpAPIkey'),
+        "api_key": os.getenv('WEBSCRAPING_API_KEY'),
         "engine": "google_jobs",
         "q": query,
         "hl": "en",
@@ -51,16 +50,16 @@ def google_job_search(job_title, city_state, start=0):
         return None
 
 def sql_dump(df, table):
-    engine = create_engine(f"postgresql://{os.getenv('MasterName')}:{os.getenv('MasterPass')}@{os.getenv('RDS_EndPoint')}:5432/postgres")
+    engine = create_engine(f"postgresql://{os.getenv('PSQL_MASTER_NAME')}:{os.getenv('PSQL_KEY')}@{os.getenv('RDS_ENDPOINT')}:5432/postgres")
     with engine.connect() as conn:
-        df.to_sql(table, conn, if_exists='append', chunksize=20, method='multi', index=False)
+        df.to_sql(table, conn, if_exists='append', chunksize=20, method='None', index=False)
         print(f"Dumped {df.shape} to SQL table {table}")
 
 def process_batch(job, city_state, start):
     df_10jobs = google_job_search(job, city_state, start)
     if df_10jobs is not None:
         print(f'City: {city_state} Job: {job} Start: {start}')
-        print(df_10jobs.shape)
+        
         date = dt.datetime.today().strftime('%Y-%m-%d')
         df_10jobs['retrieve_date'] = date
         df_10jobs.drop_duplicates(subset=['job_id', 'company_name'], inplace=True)
@@ -80,11 +79,10 @@ def main(job_list, city_state_list):
             future.result()
 
 if __name__ == "__main__":
-    job_list = ["Data Scientist", "Machine Learning Engineer", "AI Gen Engineer",
-                "Data Analyst", "Data Engineer", "Business Intelligence Analyst"]
+    job_list = ["Data Scientist", "Machine Learning Engineer", "AI Gen Engineer"]
     city_state_list = ["Atlanta, GA", "Austin, TX", "Boston, MA", "Chicago, IL", 
                     "Denver CO", "Dallas-Ft. Worth, TX", "Los Angeles, CA",
                     "New York City NY", "San Francisco, CA", "Seattle, WA",
                     "Palo Alto CA", "Mountain View CA"]
-    simple_city_state_list: list[str] = ["Palo Alto CA", "San Francisco CA", ]
+    simple_city_state_list: list[str] = ["Palo Alto CA", "San Francisco CA", "Mountain View CA", "San Jose, CA"]
     main(job_list, city_state_list)

utils/job_desc_pydantic.py

@@ -4,19 +4,11 @@ from langchain_core.pydantic_v1 import BaseModel, Field
 class CompanyOverview(BaseModel):
     """
     A model for capturing key information about the company offering the job.
-    
-    Extract relevant details about the company from the job description, 
-    including a brief overview of its industry and products, its mission and 
-    values, size, and location(s).
-    
-    Focus on capturing the most salient points that give a well-rounded picture
-    of the company and its culture.
     """
 
     about: Optional[str] = Field(
         None, 
-        description="""Brief description of the company, its industry, products, services, 
-                    and any notable achievements or differentiators"""
+        description="""Overview of the company, industry, products, services, and notable achievements"""
     )
 
     mission_and_values: Optional[str] = Field(
@@ -31,121 +23,75 @@ class CompanyOverview(BaseModel):
     
     locations: Optional[str] = Field(
         None,
-        description="""Geographic presence of the company, including headquarters, 
-                    offices, and any remote work options"""
+        description="""City, State where this position is based. """
     )
-    
-    city: Optional[str] = Field(None, description="City where the company is located")
-    
-    state: Optional[str] = Field(None, description="State where the company is located")
-
 
 class RoleSummary(BaseModel):
     """
     A model for capturing the key summary points about the job role.
     
-    Extract the essential high-level details about the role from the job description,
-    such as the job title, the team or department the role belongs to, the role type, 
-    and any remote work options.
-    
     Prioritize information that helps understand the overall scope and positioning 
     of the role within the company.
     """
     
     title: str = Field(..., description="Title of the job role")
     
-    team_or_department: Optional[str] = Field(
-        None,
+    team_or_department: Optional[str] = Field(None,
         description="""Team, department, or business unit the role belongs to, 
                     including any collaborations with other teams"""
     )
     
-    role_type: Optional[str] = Field(
-        None,
+    role_type: Optional[str] = Field(None,
         description="Type of role (full-time, part-time, contract, etc.)"
     )
     
-    remote: Optional[str] = Field(
-        None,
+    remote: Optional[str] = Field(None,
         description="Remote work options for the role (full, hybrid, none)"
     )
 
 class ResponsibilitiesAndQualifications(BaseModel):
     """
-    A model for capturing the key responsibilities, requirements, and preferred 
-    qualifications for the job role.
-
-    Extract the essential duties and expectations of the role, the mandatory 
-    educational background and experience required, and any additional skills 
-    or characteristics that are desirable but not strictly necessary.
-
-    The goal is to provide a clear and comprehensive picture of what the role 
-    entails and what qualifications the ideal candidate should possess.
+    A model for capturing the key summary points about the job role.
     """
 
-    responsibilities: List[str] = Field(
-        description="""The core duties, tasks, and expectations of the role, encompassing 
-                    areas such as metrics, theories, business understanding, product 
-                    direction, systems, leadership, decision making, strategy, and 
-                    collaboration, as described in the job description"""
+    responsibilities: List[str] = Field(None,
+        description="""List of responsibilities, including tasks, duties, and expectations for the role"""
     )
 
-    required_qualifications: List[str] = Field(
-        description="""The essential educational qualifications (e.g., Doctorate, 
-                    Master's, Bachelor's degrees in specific fields) and years of 
-                    relevant professional experience that are mandatory for the role, 
-                    including any alternative acceptable combinations of education 
-                    and experience, as specified in the job description"""
+    required_qualifications: List[str] = Field(None,
+        description="""Essential educational qualifications and professional experience required for the role.
+                    This may include, but not limited to, degrees, certifications, years of experience, technical skills, and domain knowledge.
+                    """
     )
     
-    preferred_qualifications: List[str] = Field(
-        description="""Any additional skills, experiences, characteristics, or domain 
-                    expertise that are valuable for the role but not absolute 
-                    requirements, such as proficiency with specific tools/technologies, 
-                    relevant soft skills, problem solving abilities, and industry 
-                    knowledge, as mentioned in the job description as preferred or 
-                    nice-to-have qualifications"""
+    preferred_qualifications: List[str] = Field(None,
+        description="""Any additional qualifications that a candidate may possess to stand out or excel in the role.
+                    This may include preferred skills, experience, certifications, or other attributes that are not essential but beneficial for the role.
+                    """
     )
     
 class CompensationAndBenefits(BaseModel):
     """
-    A model for capturing the compensation and benefits package for the job role.
-    
-    Extract details about the salary or pay range, bonus and equity compensation, 
-    benefits, and perks from the job description.
+    A class for capturing details about the compensation and benefits offered for the role
     
     Aim to provide a comprehensive view of the total rewards offered for the role,
     including both monetary compensation and non-monetary benefits and perks.
     """
     
-    salary_or_pay_range: Optional[str] = Field(
-        None,
-        description="""The salary range or hourly pay range for the role, including 
-                    any specific numbers or bands mentioned in the job description"""
+    salary_or_pay_range: Optional[str] = Field(None,
+        
+        description="""Salary range or hourly pay range for the role"""
     )
     
-    bonus_and_equity: Optional[str] = Field(
-        None,
-        description="""Any information about bonus compensation, such as signing bonuses, 
-                    annual performance bonuses, or other incentives, as well as details 
-                    about equity compensation like stock options or RSUs"""
+    bonus_and_equity: Optional[str] = Field(None,
+        description="""Information about bonus and equity compensation"""
     )
     
-    benefits: Optional[List[str]] = Field(
-        None,
-        description="""A list of benefits offered for the role, such as health insurance, 
-                    dental and vision coverage, retirement plans (401k, pension), paid 
-                    time off (vacation, sick days, holidays), parental leave, and any 
-                    other standard benefits mentioned in the job description"""
+    benefits_and_perks: Optional[List[str]] = Field(None,
+        description="""List of benefits and perks offered for the role, such as insurance, retirement plans, and paid time off.
+                        Can also include additional perks like free meals, wellness programs, learning stipends, etc."""
     )
     
-    perks: Optional[List[str]] = Field(
-        None,
-        description="""A list of additional perks and amenities offered, such as free food 
-                    or snacks, commuter benefits, wellness programs, learning and development 
-                    stipends, employee discounts, or any other unique perks the company 
-                    provides to its employees, as mentioned in the job description"""
-    )
 
 class JobDescription(BaseModel):
     """Extracted information from a job description."""

utils/parse_description.py

@@ -5,36 +5,30 @@ from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.pydantic_v1 import BaseModel, Field
 
 from langchain_groq import ChatGroq
+from langchain_anthropic import ChatAnthropic
 from dotenv import load_dotenv
 import os
 
 from utils.job_desc_pydantic import JobDescription
 
-load_dotenv()
 
+# Set the LANGCHAIN_TRACING_V2 environment variable to 'true'
+os.environ['LANGCHAIN_TRACING_V2'] = 'true'
+
+# Set the LANGCHAIN_PROJECT environment variable to the desired project name
+os.environ['LANGCHAIN_PROJECT'] = 'JobDescriptionProject'
 
+load_dotenv()
 
-def extract_desc_fields(raw_job_description):
+def extract_desc_fields(raw_job_description, model_name="llama3-70b-8192"):
     prompt = ChatPromptTemplate.from_messages(
         [
             (
                 "system",
                 """You are an expert at identifying key aspects of job descriptions. Your task is to extract important information from a raw job description and organize it into a structured format using the ResponsibilitiesAndQualifications class.
 
-                    When parsing the job description, your goal is to capture as much relevant information as possible in the appropriate fields of the class. This includes:
-
-                    1. All key responsibilities and duties of the role, covering the full range of tasks and expectations.
-                    2. The required educational qualifications and years of experience, including different acceptable combinations.
-                    3. Any additional preferred skills, experiences, and characteristics that are desirable for the role.
-
-                    Avoid summarizing or paraphrasing the information. Instead, extract the details as closely as possible to how they appear in the original job description. The aim is to organize and structure the raw data, not to condense or interpret it.
-
-                    Some specific things to look out for:
-                    - Responsibilities related to metrics, theories, business understanding, product direction, systems, leadership, decision making, strategy, and collaboration
-                    - Required degrees (Doctorate, Master's, Bachelor's) in relevant fields, along with the corresponding years of experience
-                    - Preferred qualifications like years of coding experience, soft skills, problem solving abilities, and domain expertise
-
-                    If any of these details are missing from the job description, simply omit them from the output rather than trying to infer or fill in the gaps.
+                    When parsing the job description, your goal is to capture as much relevant information as possible in the appropriate fields of the class. 
+                    
 
                     The structured data you extract will be used for further analysis and insights downstream, so err on the side of including more information rather than less. The key is to make the unstructured job description data more organized and manageable while still retaining all the important details.
                 """,
@@ -43,13 +37,16 @@ def extract_desc_fields(raw_job_description):
         ]
     )
     
+    # llm = ChatAnthropic(model_name="claude-3-sonnet-20240229")
     llm = ChatGroq(model_name="llama3-70b-8192")
+    # llm = ChatGroq(model_name="llama3-8b-8192")
 
     extractor = prompt | llm.with_structured_output(
         schema=JobDescription,
         method="function_calling",
         include_raw=False,
     )
-    
-    return extractor.invoke(raw_job_description)
+    clean_description = extractor.invoke(raw_job_description)
+    print(clean_description)
+    return clean_description