[FIX] Filters and search

app.py

@@ -106,14 +106,14 @@ def update_df(shown_columns, subset="datasets"):
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    query: str,
+    query: str = "",
     type_query: list = None,
-    architecture_query: list = None,
+    domain_specific_query: list = None,
     size_query: list = None,
     precision_query: str = None,
     show_deleted: bool = False,
 ):
-    filtered_df = filter_models(hidden_df, type_query, architecture_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_models(hidden_df, type_query, domain_specific_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns, list(hidden_df.columns))
     return df
@@ -157,7 +157,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, architecture_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, domain_specific_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     # if show_deleted:
@@ -168,13 +168,21 @@ def filter_models(
     filtered_df = df
 
     if type_query is not None:
-        type_emoji = [t[0] for t in type_query]
-        filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-
-    if architecture_query is not None:
-        arch_types = [t for t in architecture_query]
-        filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
-            # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
+        type_name = [t.split(" ")[1] for t in type_query]
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type.name].isin(type_name)]
+
+    if domain_specific_query is not None:
+        domain_specifics = []
+        if "Yes" in domain_specific_query:
+            domain_specifics.append(True)
+        if "No" in domain_specific_query:
+            domain_specifics.append(False)
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.is_domain_specific.name].isin(domain_specifics)]
+        
+    # if architecture_query is not None:
+    #     arch_types = [t for t in architecture_query]
+    #     filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
+    #         # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
     
     if precision_query is not None:
         if AutoEvalColumn.precision.name in df.columns:
@@ -291,6 +299,13 @@ with demo:
                     #     interactive=True,
                     #     elem_id="filter-columns-architecture",
                     # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain specific models",
+                        choices=["Yes", "No"],
+                        value=["Yes", "No"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
                     filter_columns_size = gr.CheckboxGroup(
                         label="Model sizes (in billions of parameters)",
                         choices=list(NUMERIC_INTERVALS.keys()),
@@ -311,44 +326,49 @@ with demo:
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
-            # hidden_leaderboard_table_for_search = gr.components.Dataframe(
-            #     value=datasets_original_df[DATASET_COLS],
-            #     headers=DATASET_COLS,
-            #     datatype=TYPES,
-            #     visible=False,
-            # )
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[DATASET_COLS],
+                headers=DATASET_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
 
                         
-            # search_bar.submit(
-            #     update_table,
-            #     [
-            #         hidden_leaderboard_table_for_search,
-            #         shown_columns,
-            #         search_bar,
-            #         filter_columns_type,
-            #         # filter_columns_architecture
-            #     ],
-            #     leaderboard_table,
-            # )
-            # for selector in [
-            #     shown_columns,
-            #     filter_columns_type,
-            #     # filter_columns_architecture,
-            #     # filter_columns_size,
-            #     # deleted_models_visibility,
-            # ]:
-            #     selector.change(
-            #         update_table,
-            #         [
-            #             hidden_leaderboard_table_for_search,
-            #             shown_columns,
-            #             search_bar,
-            #             filter_columns_type,
-            #             # filter_columns_architecture,
-            #         ],
-            #         leaderboard_table,
-            #         queue=True,
-            #     )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                # filter_columns_architecture,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                        # filter_columns_architecture,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
 
         with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
             pass

src/display/utils.py

@@ -28,25 +28,25 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
+auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# Scores
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True)])
-# Model information
+auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
+auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["backbone", ColumnContent, ColumnContent("Base Model", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False, True)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False)])
+# auto_eval_column_dict.append(["backbone", ColumnContent, ColumnContent("Base Model", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False, True)])
-auto_eval_column_dict.append(
-    ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, True)]
-)
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, True)])
+# auto_eval_column_dict.append(["display_result", ColumnContent, ColumnContent("Display Result", "bool", False, True)])
+auto_eval_column_dict.append(["date", ColumnContent, ColumnContent("Submission Date", "str", False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -77,7 +77,7 @@ class ModelType(Enum):
     # FINETUNED = ModelDetails(name="fine-tuned", symbol="⚪")
     PT = ModelDetails(name="pretrained", symbol="🟢")
     # FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    # DS = ModelDetails(name="domain-specific", symbol="➕")
+    # DS = ModelDetails(name="domain-specific", symbol="🏥")
     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
     RL = ModelDetails(name="preference-tuned", symbol="🟦")
     Unknown = ModelDetails(name="", symbol="?")
@@ -99,7 +99,7 @@ class ModelType(Enum):
             return ModelType.RL
         if "instruction-tuned" in type or "⭕" in type:
             return ModelType.IFT
-        # if "domain-specific" in type or "➕" in type:
+        # if "domain-specific" in type or "🏥" in type:
         #     return ModelType.DS
         return ModelType.Unknown
 
@@ -129,7 +129,16 @@ class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
-
+    Unknown = ModelDetails("?")
+    
+    def from_str(wt):
+        if "original" in wt.lower():
+            return WeightType.Original
+        if "adapter" in wt.lower():
+            return WeightType.Adapter
+        if "delta" in wt.lower():
+            return WeightType.Delta
+        return WeightType.Unknown
 
 class Precision(Enum):
     auto = ModelDetails("auto")

src/leaderboard/read_evals.py

@@ -22,11 +22,12 @@ class EvalResult:
     model: str
     revision: str  # commit hash, "" if main
     dataset_results: dict
+    is_domain_specific: bool
+    use_chat_template: bool
     # clinical_type_results:dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original  # Original or Adapter
-    architecture: str = "Unknown"
     backbone:str = "Unknown"
     license: str = "?"
     likes: int = 0
@@ -104,17 +105,20 @@ class EvalResult:
             full_model=full_model,
             org=org,
             model=model,
+            revision=config.get("revision", ""),
             dataset_results=dataset_results,
-            # clinical_type_results=types_results,
+            is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
+            use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
-            revision=config.get("revision", ""),
-            still_on_hub=still_on_hub,
-            # architecture=model_architecture,
-            backbone=backbone,
             model_type=model_type,
-            num_params=num_params,
+            weight_type=WeightType.from_str(config.get("weight_type", "")),  # Assuming the default value
+            backbone=backbone,
             license=license,
-            display_result=display_result
+            likes=config.get("likes", 0),  # Assuming a default value
+            num_params=num_params,
+            still_on_hub=still_on_hub,
+            display_result=display_result,
+            date=config.get("submitted_time","")
         )
 
     def update_with_request_file(self, requests_path):
@@ -146,17 +150,20 @@ class EvalResult:
                 "eval_name": self.eval_name,  # not a column, just a save name,
                 AutoEvalColumn.precision.name: self.precision.value.name,
                 AutoEvalColumn.model_type.name: self.model_type.value.name,
-                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol + (" 🏥" if self.is_domain_specific else ""),
                 AutoEvalColumn.weight_type.name: self.weight_type.value.name,
                 # AutoEvalColumn.architecture.name: self.architecture.value.name,
-                AutoEvalColumn.backbone.name: self.backbone,
+                # AutoEvalColumn.backbone.name: self.backbone,
                 AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+                AutoEvalColumn.is_domain_specific.name: self.is_domain_specific,
+                AutoEvalColumn.use_chat_template.name: self.use_chat_template,
                 AutoEvalColumn.revision.name: self.revision,
                 AutoEvalColumn.average.name: average,
                 AutoEvalColumn.license.name: self.license,
                 AutoEvalColumn.likes.name: self.likes,
                 AutoEvalColumn.params.name: self.num_params,
                 AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+                AutoEvalColumn.date.name: self.date,
                 "display_result" : self.display_result,
             }
 

src/populate.py

@@ -10,7 +10,7 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path, evaluation_metric)
+    raw_data =  get_raw_eval_results(results_path, requests_path, evaluation_metric)
     # print(raw_data)
     # raise Exception("stop")
     all_data_json = [v.to_dict(subset=subset) for v in raw_data]