added evaluation metric type radio button

app.py

@@ -60,12 +60,20 @@ try:
 except Exception:
     restart_space()
 
+# Span based results
+_, span_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "SpanBasedWithPartialOverlap", "datasets")
+span_based_datasets_leaderboard_df = span_based_datasets_original_df.copy()
 
-raw_data, datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "datasets")
-datasets_leaderboard_df = datasets_original_df.copy()
+_, span_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "SpanBasedWithPartialOverlap", "clinical_types")
+span_based_types_leaderboard_df = span_based_types_original_df.copy()
+
+# Token based results
+_, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
+token_based_datasets_leaderboard_df = token_based_datasets_original_df.copy()
+
+_, token_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "clinical_types")
+token_based_types_leaderboard_df = token_based_types_original_df.copy()
 
-raw_data, types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "clinical_types")
-types_leaderboard_df = types_original_df.copy()
 
 (
     finished_eval_queue_df,
@@ -74,6 +82,36 @@ types_leaderboard_df = types_original_df.copy()
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
+def update_df(evaluation_metric, shown_columns, subset="datasets"):
+    print(evaluation_metric)
+
+    if subset == "datasets":
+        match evaluation_metric:
+            case "Span Based":
+                leaderboard_table_df = span_based_datasets_leaderboard_df.copy()
+                hidden_leader_board_df = span_based_datasets_original_df
+            case "Token Based":
+                leaderboard_table_df = token_based_datasets_leaderboard_df.copy()
+                hidden_leader_board_df = token_based_datasets_original_df
+            case _:
+                pass
+    else:
+        match evaluation_metric:
+            case "Span Based":
+                leaderboard_table_df = span_based_types_leaderboard_df.copy()
+                hidden_leader_board_df = span_based_types_original_df
+            case "Token Based":
+                leaderboard_table_df = token_based_types_leaderboard_df.copy()
+                hidden_leader_board_df = token_based_types_original_df
+            case _:
+                pass        
+    
+
+    value_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns
+
+    return leaderboard_table_df[value_cols], hidden_leader_board_df
+
+
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
@@ -249,6 +287,12 @@ with demo:
                     #     )
                 with gr.Column(min_width=320):
                     # with gr.Box(elem_id="box-filter"):
+
+                    eval_metric = gr.Radio(
+                            choices=["Span Based", "Token Based"],
+                            value = "Span Based",
+                            label="Evaluation Metric",
+                        )
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model Types",
                         choices=[t.to_str() for t in ModelType],
@@ -270,6 +314,9 @@ with demo:
                     #     interactive=True,
                     #     elem_id="filter-columns-size",
                     # )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(eval_metric.value, shown_columns.value, subset="datasets")
+
             leaderboard_table = gr.components.Dataframe(
                 value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
@@ -286,6 +333,19 @@ with demo:
                 datatype=TYPES,
                 visible=False,
             )
+
+            eval_metric.change(
+                lambda a, b: update_df(a,b, "datasets") , 
+                inputs=[
+                    eval_metric,
+                    shown_columns,
+                ], 
+                outputs=[
+                        leaderboard_table, 
+                        hidden_leaderboard_table_for_search,
+                        ]
+                )
+                        
             search_bar.submit(
                 update_table,
                 [
@@ -317,6 +377,7 @@ with demo:
                     queue=True,
                 )
 
+            
         with gr.TabItem("🏅 Clinical Types", elem_id="llm-benchmark-tab-table", id=4):
             with gr.Row():
                 with gr.Column():
@@ -343,6 +404,12 @@ with demo:
                     #         value=False, label="Show gated/private/deleted models", interactive=True
                     #     )
                 with gr.Column(min_width=320):
+
+                    eval_metric = gr.Radio(
+                            choices=["Span Based", "Token Based"],
+                            value = "Span Based",
+                            label="Evaluation Metric",
+                        )                    
                     # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model Types",
@@ -372,6 +439,7 @@ with demo:
                     #     interactive=True,
                     #     elem_id="filter-columns-size",
                     # )
+            types_leaderboard_df, types_original_df = update_df(eval_metric.value, shown_columns.value, subset="clinical_types")
 
             leaderboard_table = gr.components.Dataframe(
                 value=types_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
@@ -389,6 +457,19 @@ with demo:
                 datatype=TYPES,
                 visible=False,
             )
+
+            eval_metric.change(
+                fn=lambda a, b: update_df(a,b, "clinical_types"), 
+                inputs=[
+                    eval_metric,
+                    shown_columns,
+                ], 
+                outputs=[
+                        leaderboard_table, 
+                        hidden_leaderboard_table_for_search
+                        ]
+                    )
+                        
             search_bar.submit(
                 update_table,
                 [

src/display/utils.py

@@ -162,6 +162,9 @@ class PromptTemplateName(Enum):
     LLamaNERTemplate = "llama_70B_ner"
     # MixtralNERTemplate = "mixtral_ner_v0.3"
 
+class EvaluationMetrics(Enum):
+    SpanBased = "Span Based"
+    TokenBased = "Token Based"
 
 
 # Column selection

src/leaderboard/read_evals.py

@@ -36,7 +36,7 @@ class EvalResult:
     display_result:bool = True
 
     @classmethod
-    def init_from_json_file(self, json_filepath):
+    def init_from_json_file(self, json_filepath, evaluation_metric):
         """Inits the result from the specific model result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
@@ -82,7 +82,7 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["dataset_results"].items() if task.benchmark == k])
+            accs = np.array([v.get(task.metric, None) for k, v in data[evaluation_metric]["dataset_results"].items() if task.benchmark == k])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
@@ -94,7 +94,7 @@ class EvalResult:
             clinical_type = clinical_type.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(clinical_type.metric, None) for k, v in data["clinical_type_results"].items() if clinical_type.benchmark == k])
+            accs = np.array([v.get(clinical_type.metric, None) for k, v in data[evaluation_metric]["clinical_type_results"].items() if clinical_type.benchmark == k])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
@@ -212,7 +212,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metric: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
@@ -233,7 +233,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result = EvalResult.init_from_json_file(model_result_filepath, evaluation_metric)
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together

src/populate.py

@@ -8,9 +8,9 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, subset:str) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(results_path, requests_path, evaluation_metric)
     all_data_json = [v.to_dict(subset=subset) for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)