added verified agents management and column and fixed widths

about.md

@@ -1,4 +1 @@
-
-# Agent leaderboard
-
 Coming soon...

app.py

@@ -227,15 +227,12 @@ with gr.Blocks() as demo:
                     Leaderboard(
                         value=parse_json_files(os.path.join(abs_path, "evals_live"), 'usaco'),
                         select_columns=SelectColumns(
-                            default_selection=config.USACO_ON_LOAD_COLUMNS,
+                            default_selection=config.USACO_ON_LOAD_COLUMNS + ["Verified"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
                         hide_columns=config.USACO_HIDE_COLUMNS,
                         # search_columns=config.USACO_SEARCH_COLUMNS,
-                        column_widths={"Agent Name": 40,
-                                       "Accuracy": 20,
-                                       "Total Cost": 20},
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'usaco'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
@@ -359,15 +356,12 @@ with gr.Blocks() as demo:
                     Leaderboard(
                         value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'),
                         select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
                         hide_columns=config.SWEBENCH_HIDE_COLUMNS,
-                        # search_columns=config.SWEBENCH_SEARCH_COLUMNS,
-                        column_widths={"Agent Name": 40,
-                                       "Accuracy": 20,
-                                       "Total Cost": 20},
+                        # search_columns=config.SWEBENCH_SEARCH_COLUMNS
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
@@ -490,15 +484,12 @@ with gr.Blocks() as demo:
                     Leaderboard(
                         value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite'),
                         select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
                         # search_columns=config.SWEBENCH_SEARCH_COLUMNS,
-                        hide_columns=config.SWEBENCH_HIDE_COLUMNS,
-                        column_widths={"Agent Name": 40,
-                                       "Accuracy": 20,
-                                       "Total Cost": 20},
+                        hide_columns=config.SWEBENCH_HIDE_COLUMNS
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
@@ -621,15 +612,12 @@ with gr.Blocks() as demo:
                     Leaderboard(
                         value=parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench'),
                         select_columns=SelectColumns(
-                            default_selection=config.MLAGENTBENCH_ON_LOAD_COLUMNS,
+                            default_selection=config.MLAGENTBENCH_ON_LOAD_COLUMNS + ["Verified"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
                         # search_columns=config.MLAGENTBENCH_SEARCH_COLUMNS,
                         hide_columns=config.MLAGENTBENCH_HIDE_COLUMNS,
-                        column_widths={"Agent Name": 40,
-                                       "Overall Score": 20,
-                                       "Total Cost": 20},
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench'), "Total Cost", "Overall Score", "Total Cost (in USD)", "Overall Score", ["Agent Name"]))

utils/db.py

@@ -6,6 +6,7 @@ from functools import lru_cache
 import threading
 import pandas as pd
 import ast
+import yaml
 
 class TracePreprocessor:
     def __init__(self, db_path='preprocessed_traces.db'):
@@ -160,6 +161,12 @@ class TracePreprocessor:
             '''
             df = pd.read_sql_query(query, conn, params=(benchmark_name,))
 
+        # Load verified agents
+        verified_agents = self.load_verified_agents()
+
+        # Add 'Verified' column
+        df['Verified'] = df.apply(lambda row: '✓' if (benchmark_name, row['agent_name']) in verified_agents else '', axis=1)
+
         # Round float columns to 3 decimal places
         float_columns = ['total_cost', 'accuracy', 'precision', 'recall', 'f1_score', 'auc', 'overall_score', 'vectorization_score', 'fathomnet_score', 'feedback_score', 'house-price_score', 'spaceship-titanic_score', 'amp-parkinsons-disease-progression-prediction_score', 'cifar10_score', 'imdb_score']
         for column in float_columns:
@@ -229,6 +236,17 @@ class TracePreprocessor:
         })
 
         return df
+    
+    def load_verified_agents(self, file_path='verified_agents.yaml'):
+        with open(file_path, 'r') as f:
+            verified_data = yaml.safe_load(f)
+        
+        verified_agents = set()
+        for benchmark, agents in verified_data.items():
+            for agent in agents:
+                verified_agents.add((benchmark, agent['agent_name']))
+        
+        return verified_agents
 
 if __name__ == '__main__':
     preprocessor = TracePreprocessor()

verified_agents.yaml

@@ -0,0 +1,31 @@
+# This file contains information about verified agent results for different benchmarks.
+# Format: 
+#   benchmark_name:
+#     - agent_name: "Name of the agent"
+#       verification_date: YYYY-MM-DD
+
+usaco:
+  - agent_name: "USACO Reflexion + Episodic (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-20
+  - agent_name: "USACO Reflexion + Episodic + Semantic (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-20
+  - agent_name: "USACO Reflexion (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-20
+  - agent_name: "USACO Episodic (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-12
+  - agent_name: "USACO Reflexion + Semantic (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-20
+  - agent_name: "USACO Zero-shot (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-11
+  - agent_name: "USACO Semantic (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-12
+
+swebench_verified:
+  - agent_name: "Agentless (gpt-4o-mini-2024-07-18) (50 Instances)"
+    verification_date: 2024-08-17
+  - agent_name: "SWE-agent (gpt-4o-mini-2024-07-18) (Cost Limit: $1) (50 Instances)"
+    verification_date: 2024-08-19
+
+mlagentbench:
+  - agent_name: "MLAgentBench ResearchAgent (gpt-4o-mini-2024-07-18)"
+    verification_date: 2024-08-19