[ADD] Med Safety

app.py

@@ -19,11 +19,14 @@ from src.about import (
     LOGO
 )
 from src.display.css_html_js import custom_css
+# changes to be made here
 from src.display.utils import (
     DATASET_BENCHMARK_COLS,
     OPEN_ENDED_BENCHMARK_COLS,
+    MED_SAFETY_BENCHMARK_COLS,
     DATASET_COLS,
     OPEN_ENDED_COLS,
+    MED_SAFETY_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -61,12 +64,17 @@ except Exception:
     restart_space()
 
 # Span based results
+# changes to be made here
+
 _, harness_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "accuracy", "datasets")
 harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
 
 _, open_ended_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OPEN_ENDED_COLS, OPEN_ENDED_BENCHMARK_COLS, "score", "open_ended")
 open_ended_leaderboard_df = open_ended_original_df.copy()
 
+_, med_safety_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MED_SAFETY_COLS, MED_SAFETY_BENCHMARK_COLS, "score", "med_safety")
+med_safety_leaderboard_df = med_safety_original_df.copy()
+
 # breakpoint()
 # # Token based results
 # _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
@@ -84,12 +92,16 @@ open_ended_leaderboard_df = open_ended_original_df.copy()
 
 
 def update_df(shown_columns, subset="datasets"):
+    # changes to be made here
     if subset == "datasets":
         leaderboard_table_df = harness_datasets_leaderboard_df.copy()
         hidden_leader_board_df = harness_datasets_original_df
     elif subset == "open_ended":
         leaderboard_table_df = open_ended_leaderboard_df.copy()
         hidden_leader_board_df = open_ended_original_df
+    elif subset == "med_safety":
+        leaderboard_table_df = med_safety_leaderboard_df.copy()
+        hidden_leader_board_df = med_safety_original_df
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -432,10 +444,116 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
-
         with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown("# Coming Soon!!!", elem_classes="markdown-text")
-            pass
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain specific models",
+                        choices=["Yes", "No"],
+                        value=["Yes", "No"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[MED_SAFETY_COLS],
+                headers=MED_SAFETY_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
         with gr.TabItem("🏅 Cross Examination", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown("# Coming Soon!!!", elem_classes="markdown-text")
             pass

src/about.py

@@ -37,6 +37,25 @@ class OpenEndedColumns(Enum):
     column0 = OpenEndedColumn("ELO", "score", "ELO") 
     column1 = OpenEndedColumn("Score", "score", "Score") 
 
+# changes to be made here
+
+@dataclass
+class MedSafetyColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedSafetyColumns(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    med_safety_column0 = MedSafetyColumn("Competence, Compassion, and Respect for Human Dignity", "score", "Competence, Compassion, and Respect for Human Dignity") 
+    med_safety_column1 = MedSafetyColumn("Patient Rights and Confidentiality", "score", "Patient Rights and Confidentiality") 
+    med_safety_column2 = MedSafetyColumn("Continued Study and Information Sharing", "score", "Continued Study and Information Sharing") 
+    med_safety_column3 = MedSafetyColumn("Medical Care for All", "score", "Medical Care for All") 
+    med_safety_column4 = MedSafetyColumn("Community and Public Health", "score", "Community and Public Health") 
+    med_safety_column5 = MedSafetyColumn("Physician's Freedom of Choice", "score", "Physician's Freedom of Choice") 
+    med_safety_column6 = MedSafetyColumn("Professionalism and Honesty", "score", "Professionalism and Honesty") 
+    med_safety_column7 = MedSafetyColumn("Responsibility to Patient", "score", "Responsibility to Patient") 
+    med_safety_column8 = MedSafetyColumn("Law and Responsibility to Society", "score", "Law and Responsibility to Society") 
 
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -3,7 +3,8 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import HarnessTasks, OpenEndedColumns
+# changes to be made here
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns
 
 
 def fields(raw_class):
@@ -15,6 +16,7 @@ def fields(raw_class):
 # when a modif is needed
 @dataclass
 class ColumnContent:
+    # changes to be made here
     name: str
     type: str
     displayed_by_default: bool
@@ -34,11 +36,14 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, invariant=False)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, med_safety_col=True, invariant=False)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
 for column in OpenEndedColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_col=True, invariant=False)])
+# changes to be made here
+for column in MedSafetyColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, med_safety_col=True, invariant=False)])
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -57,6 +62,7 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 
 
 ## For the queue columns in the submission tab
+# changes to be made here
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
@@ -67,6 +73,7 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     closed_ended_status = ColumnContent("closed_ended_status", "str", True)
     open_ended_status = ColumnContent("open_ended_status", "str", True)
+    med_safety_status = ColumnContent("med_safety_status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -185,10 +192,15 @@ class EvaluationMetrics(Enum):
 
 
 # Column selection
-DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
-OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
-MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.dataset_task_col and not c.cross_examination_col]
-CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.dataset_task_col]
+# changes to be made here
+DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.dataset_task_col or c.invariant)]
+OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_col or c.invariant)]
+MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.med_safety_col or c.invariant)]
+CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
+# DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
+# OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
+# MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.dataset_task_col and not c.cross_examination_col]
+# CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.dataset_task_col]
 
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
@@ -197,9 +209,10 @@ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default a
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
+# changes to be made here
 DATASET_BENCHMARK_COLS = [t.value.col_name for t in HarnessTasks]
 OPEN_ENDED_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedColumns]
-# MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyTasks]
+MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyColumns]
 # CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
 
 NUMERIC_INTERVALS = {

src/leaderboard/read_evals.py

@@ -8,7 +8,8 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns
+# changes to be made here
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -22,6 +23,7 @@ class EvalResult:
     model: str
     revision: str  # commit hash, "" if main
     dataset_results: dict
+    # changes to be made here
     open_ended_results: dict
     med_safety_results: dict
     cross_examination_results: dict
@@ -104,7 +106,19 @@ class EvalResult:
                 mean_acc = np.mean(accs)  # * 100.0
                 open_ended_results[task.benchmark] = mean_acc
         # breakpoint()
+        # changes to be made here
         med_safety_results = {}
+        if "med-safety" in data["results"]:
+            for task in MedSafetyColumns:
+                task = task.value
+                try:
+                    accs = np.array([v.get(task.metric, None) for k, v in data["results"]["med-safety"].items() if task.benchmark == k])
+                except:
+                    accs = np.array([])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+                mean_acc = np.mean(accs)  # * 100.0
+                med_safety_results[task.benchmark] = mean_acc
         cross_examination_results = {}
         # types_results = {}
         # for clinical_type in ClinicalTypes:
@@ -198,12 +212,14 @@ class EvalResult:
                 for task in OpenEndedColumns:
                     data_dict[task.value.col_name] = self.open_ended_results[task.value.benchmark]
             return data_dict
-        
-        # if subset == "med_safety":
-        #     if len(self.med_safety_results) > 0:
-        #         for task in MedSafetyTasks:
-        #             data_dict[task.value.col_name] = self.med_safety_results[task.value.benchmark]
-        #     return data_dict
+        # changes to be made here
+        if subset == "med_safety":
+            average = sum([v for v in self.med_safety_results.values() if v is not None]) / len(MedSafetyColumns)
+            data_dict[AutoEvalColumn.average.name] = average
+            if len(self.med_safety_results) > 0:
+                for task in MedSafetyColumns:
+                    data_dict[task.value.col_name] = self.med_safety_results[task.value.benchmark]
+            return data_dict
         
         # if subset == "cross_examination":
         #     if len(self.cross_examination_results) > 0:

src/populate.py

@@ -4,7 +4,8 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns
+# changes to be made here
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
@@ -16,13 +17,15 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict(subset=subset) for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
+    # changes to be made here
     if subset == "datasets":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    elif subset == "med_safety":
+        df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=True)
     elif subset == "open_ended":
         df = df.sort_values(by=["ELO"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
-
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return raw_data, df
@@ -39,8 +42,10 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data = json.load(fp)
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            # changes to be made here
             data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
             data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
+            data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
@@ -54,12 +59,14 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
                 data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
+                data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
                 all_evals.append(data)
     # breakpoint()
     pending_list = []
     running_list = []
     finished_list = []
     for run in all_evals:
+        # changes to be made here
         status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["cross-examination"]]
         status_list = status_list[:2]
         if "RUNNING" in status_list: