updated prompts

agent_monitor/monitor.py

@@ -99,23 +99,28 @@ async def analyze_task(calls, llm_client):
 
 async def analyze_step(call, step_number, total_steps, llm_client):
     prompt = f"""
-            Analyze Step {step_number}/{total_steps} of AI agent task:
-            Input: {call['inputs']}
-            Output: {call['outputs']}
-            Exception: {call['exception']}
-            Summary: {call['summary']}
-
-            Provide an analysis with:
-            1. A brief description of the agent's action.
-            2. Classify the action as one of: 'plan', 'tool', 'retrieve', or 'other'.
-            3. Give a brief evaluation of progress, obstacles, or errors.
-            4. Indicate if the agent successfully completed its intended action.
-            5. Write a concise headline summarizing the agent's action that is ideally less than 7 words long.
-
-            Ensure accuracy and conciseness. Be specific and avoid too high-level descriptions.
-            """
-
-    system_message = "You are an expert AI system analyst, skilled in categorizing and evaluating AI agent actions."
+    Analyze Step {step_number}/{total_steps} of the AI agent's USACO task solution:
+    Input: {call['inputs']}
+    Output: {call['outputs']}
+    Exception: {call['exception']}
+    Summary: {call['summary']}
+
+    Provide a detailed, technical analysis with the following:
+    1. Specific Description: Describe precisely what the agent did in this step, including any algorithms, data structures, or problem-solving techniques employed.
+    2. Action Classification: Categorize the action as one of:
+       - 'plan': Strategizing or outlining an approach
+       - 'tool': Using a specific programming construct or algorithm
+       - 'retrieve': Accessing or utilizing external information
+       - 'other': Any action that doesn't fit the above categories
+    3. Technical Evaluation: Assess the technical merit of the agent's approach. Comment on efficiency, correctness, and adherence to USACO problem-solving best practices.
+    4. Success: Determine if the agent successfully completed its intended action.
+    5. Concise Headline: Write a technically precise headline (max 7 words) that captures the essence of this step.
+
+    Your analysis should be highly specific to this task. Avoid generalities and focus on the technical details of the agent's approach to this particular problem.
+    """
+
+    system_message = "You are an expert in AI agent design and evaluation. Analyze the AI agent's actions with the depth and specificity expected in a detailed expert review. Focus on providing insights that would be valuable to an AI researcher specializing in AI agent development."
+
     analysis = await llm_client.generate_text(prompt, system_message, response_format=StepAnalysis)
 
     try:
@@ -138,20 +143,20 @@ async def summarize_task(steps, llm_client):
     steps_summary = "\n".join([f"Step {i+1}: {step['analysis']}" for i, step in enumerate(steps)])
     
     prompt = f"""
-        Analyze the following AI agent task steps:
+    Provide a comprehensive analysis of the AI agent's approach to solving this USACO task:
 
-        {steps_summary}
+    {steps_summary}
 
-        Provide a summary with:
-        1. A concise overview of the agent's approach.
-        2. Main achievements or breakthroughs.
-        3. Primary obstacles or errors encountered.
-        4. A brief evaluation of the agent's overall performance.
+    Your analysis should include:
+    1. Technical Overview: Describe the agent's overall problem-solving strategy, highlighting specific actions and techniques used throughout the task.
+    2. Key Achievements: Identify and explain the most significant breakthroughs or efficient implementations demonstrated by the agent.
+    3. Technical Challenges: Analyze the primary obstacles encountered, focusing on difficulties or conceptual misunderstandings in the context of the task.
+    4. Performance Evaluation: Assess the agent's overall performance, considering factors such as time complexity, space efficiency, code quality, and adherence to competitive programming best practices.
 
-        Focus on patterns in the agent's approach and effectiveness. Be concise and insightful.
-        """
+    Your summary should be highly technical and specific to this task. Assume the reader is an expert as well and familiar with the task context. Focus on providing insights that would be valuable to an AI researcher specializing in AI agent development.
+    """
 
-    system_message = "You are an expert AI performance analyst, skilled in evaluating and summarizing AI agent task execution."
+    system_message = "You are an expert AI performance analyst, skilled in evaluating and summarizing AI agent task execution. You are specialized in providing analyses to support AI researchers to develop AI agents."
     analysis = await llm_client.generate_text(prompt, system_message, response_format=TaskSummary)
     return json.loads(analysis)