Upload 2 files

utils/db.py

@@ -181,7 +181,7 @@ class TracePreprocessor:
         # if there is a failure report, return the first one
         return pickle.loads(df['failure_report'][0])
     
-    def _calculate_ci(self, data, confidence=0.95):
+    def _calculate_ci(self, data, confidence=0.95, type='minmax'):
         data = data[np.isfinite(data)]
 
         if len(data) < 2:
@@ -189,9 +189,15 @@ class TracePreprocessor:
         n = len(data)
 
         mean = np.mean(data)
-        sem = stats.sem(data)
 
-        ci = stats.t.interval(confidence, n-1, loc=mean, scale=sem)
+        if type == 't':
+            sem = stats.sem(data)
+            ci = stats.t.interval(confidence, n-1, loc=mean, scale=sem)
+
+        elif type == 'minmax':
+            min = np.min(data)
+            max = np.max(data)
+            ci = (min, max)
         return mean, ci[0], ci[1]
     
     def get_parsed_results(self, benchmark_name, aggregate=True):
@@ -222,12 +228,15 @@ class TracePreprocessor:
             agent_df = df[df['agent_name'] == agent_name]
             
             if len(agent_df) > 1:
-                accuracy_mean, accuracy_lower, accuracy_upper = self._calculate_ci(agent_df['accuracy'])
-                cost_mean, cost_lower, cost_upper = self._calculate_ci(agent_df['total_cost'])
+                accuracy_mean, accuracy_lower, accuracy_upper = self._calculate_ci(agent_df['accuracy'], type='minmax')
+                cost_mean, cost_lower, cost_upper = self._calculate_ci(agent_df['total_cost'], type='minmax')
                 
                 # format the confidence interval with +/- sign
-                accuracy_ci = f"± {abs(accuracy_mean - accuracy_lower):.3f}"
-                cost_ci = f"± {abs(cost_mean - cost_lower):.3f}"
+                # accuracy_ci = f"± {abs(accuracy_mean - accuracy_lower):.3f}"
+                # cost_ci = f"± {abs(cost_mean - cost_lower):.3f}"
+
+                accuracy_ci = f"-{abs(accuracy_mean - accuracy_lower):.3f}/+{abs(accuracy_mean - accuracy_upper):.3f}"
+                cost_ci = f"-{abs(cost_mean - cost_lower):.3f}/+{abs(cost_mean - cost_upper):.3f}"
                 
                 df.loc[df['agent_name'] == agent_name, 'acc_ci'] = accuracy_ci
                 df.loc[df['agent_name'] == agent_name, 'cost_ci'] = cost_ci

utils/viz.py

@@ -202,8 +202,8 @@ def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str =
 
         if len(agent_data) > 1:
             # Calculate 95% confidence intervals
-            ci_x = stats.t.interval(0.95, len(agent_data[x])-1, loc=np.mean(agent_data[x]), scale=stats.sem(agent_data[x]))
-            ci_y = stats.t.interval(0.95, len(agent_data[y])-1, loc=np.mean(agent_data[y]), scale=stats.sem(agent_data[y]))
+            # ci_x = stats.t.interval(0.95, len(agent_data[x])-1, loc=np.mean(agent_data[x]), scale=stats.sem(agent_data[x]))
+            # ci_y = stats.t.interval(0.95, len(agent_data[y])-1, loc=np.mean(agent_data[y]), scale=stats.sem(agent_data[y]))
 
             # # Add error bars for x (cost)
             # fig.add_trace(go.Scatter(