Big update

README.md

@@ -4,11 +4,12 @@ emoji: 📉
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 3.43.2
+sdk_version: 4.9.0
 app_file: app.py
 pinned: true
 license: apache-2.0
 duplicated_from: HuggingFaceH4/open_llm_leaderboard
+fullWidth: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,106 +1,69 @@
-import json
-import os
-from datetime import datetime, timezone
-import re
-from distutils.util import strtobool
-
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi, snapshot_download
+from huggingface_hub import snapshot_download
+from gradio_space_ci import configure_space_ci # FOR CI
 
-from src.assets.css_html_js import custom_css, get_window_url_params
-from src.assets.text_content import (
+from src.display.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
+    FAQ_TEXT,
     TITLE,
     BOTTOM_LOGO,
 )
-from src.display_models.get_model_metadata import DO_NOT_SUBMIT_MODELS, ModelType
-from src.display_models.utils import (
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
     AutoEvalColumn,
-    EvalQueueColumn,
+    ModelType,
     fields,
-    styled_error,
-    styled_message,
-    styled_warning,
+    WeightType,
+    Precision
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+from src.tools.collections import update_collections
+from src.tools.plots import (
+    create_metric_plot_obj,
+    create_plot_df,
+    create_scores_df,
 )
-from src.load_from_hub import get_all_requested_models, get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub
-from src.rate_limiting import user_submission_permission
-
-pd.set_option("display.precision", 1)
-
-# clone / pull the lmeh eval data
-H4_TOKEN = os.environ.get("H4_TOKEN", None)
-
-QUEUE_REPO = "open-ko-llm-leaderboard/requests"
-RESULTS_REPO = "open-ko-llm-leaderboard/results"
-
-PRIVATE_QUEUE_REPO = "open-ko-llm-leaderboard/private-requests"
-PRIVATE_RESULTS_REPO = "open-ko-llm-leaderboard/private-results"
-
-IS_PUBLIC = bool(strtobool(os.environ.get("IS_PUBLIC", "True")))
-
-EVAL_REQUESTS_PATH = "eval-queue"
-EVAL_RESULTS_PATH = "eval-results"
-
-EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
-EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
-
-api = HfApi(token=H4_TOKEN)
 
 
 def restart_space():
-    api.restart_space(repo_id="upstage/open-ko-llm-leaderboard", token=H4_TOKEN)
-
-# Rate limit variables
-RATE_LIMIT_PERIOD = 7
-RATE_LIMIT_QUOTA = 5
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-
-if not IS_PUBLIC:
-    COLS.insert(2, AutoEvalColumn.precision.name)
-    TYPES.insert(2, AutoEvalColumn.precision.type)
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [
-    c.name
-    for c in [
-        AutoEvalColumn.arc,
-        AutoEvalColumn.hellaswag,
-        AutoEvalColumn.mmlu,
-        AutoEvalColumn.truthfulqa,
-        AutoEvalColumn.commongen_v2,
-        # TODO: Uncomment when we have results for these
-        # AutoEvalColumn.ethicalverification,
-    ]
-]
-
-snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None)
-snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None)
-requested_models, users_to_submission_dates = get_all_requested_models(EVAL_REQUESTS_PATH)
+    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
-original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-models = original_df["model_name_for_query"].tolist() # needed for model backlinks in their to the leaderboard
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
+except Exception:
+    restart_space()
 
-# Commented out because it causes infinite restart loops in local
-# to_be_dumped = f"models = {repr(models)}\n"
 
-# with open("models_backlinks.py", "w") as f:
-#     f.write(to_be_dumped)
+raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+if REPO_ID == "upstage/open-ko-llm-leaderboard": # update only when it's from real leaderboard
+    update_collections(original_df.copy())
+leaderboard_df = original_df.copy()
 
-# print(to_be_dumped)
+plot_df = create_plot_df(create_scores_df(raw_data))
 
 (
     finished_eval_queue_df,
@@ -109,142 +72,34 @@ models = original_df["model_name_for_query"].tolist() # needed for model backlin
     failed_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-## INTERACTION FUNCTIONS
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    private: bool,
-    weight_type: str,
-    model_type: str,
-):
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    num_models_submitted_in_period = user_submission_permission(model, users_to_submission_dates, RATE_LIMIT_PERIOD)
-    if num_models_submitted_in_period > RATE_LIMIT_QUOTA:
-        error_msg = f"Organisation or user `{model.split('/')[0]}`"
-        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
-        error_msg += f"in the last {RATE_LIMIT_PERIOD} days.\n"
-        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
-        return styled_error(error_msg)
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # check the model actually exists before adding the eval
-    if revision == "":
-        revision = "main"
-
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error = is_model_on_hub(base_model, revision)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error = is_model_on_hub(model, revision)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-    
-    model_info = api.model_info(repo_id=model, revision=revision)
-
-    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except AttributeError:
-        try:
-            size_match = re.search(size_pattern, model.lower())
-            model_size = size_match.group(0)
-            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return 65
-
-    size_factor = 8 if (precision == "GPTQ" or "GPTQ" in model) else 1
-    model_size = size_factor * model_size
-
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        license = "?"
-
-    print("adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "private": private,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-    }
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
-
-    if user_name == "upstage":
-        return styled_warning("The model participating as a Host in Upstage does not conduct evaluations to ensure the transparency and fairness of the leaderboard. Please take this into consideration.")
-        
-    # Check if the model has been forbidden:
-    if out_path.split("eval-queue/")[1] in DO_NOT_SUBMIT_MODELS:
-        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in requested_models:
-        return styled_warning("This model has been already submitted.")
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    api.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
-
-
-# Basics
-def change_tab(query_param: str):
-    query_param = query_param.replace("'", '"')
-    query_param = json.loads(query_param)
-
-    if isinstance(query_param, dict) and "tab" in query_param and query_param["tab"] == "evaluation":
-        return gr.Tabs.update(selected=1)
-    else:
-        return gr.Tabs.update(selected=0)
-
 
 # Searching and filtering
-def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    if query != "":
-        filtered_df = search_table(filtered_df, query)
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    type_query: list,
+    precision_query: str,
+    size_query: list,
+    show_deleted: bool,
+    show_merges: bool,
+    show_flagged: bool,
+    query: str,
+):
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_merges, show_flagged)
+    filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
-
     return df
 
+
+def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
+    query = request.query_params.get("query") or ""
+    return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
+
+
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
 
+
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
@@ -256,18 +111,29 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     ]
     return filtered_df
 
-NUMERIC_INTERVALS = {
-    "Unknown": pd.Interval(-1, 0, closed="right"),
-    "0~3B": pd.Interval(0, 3, closed="right"),
-    "3~7B": pd.Interval(3, 7.3, closed="right"),
-    "7~13B": pd.Interval(7.3, 13, closed="right"),
-    "13~35B": pd.Interval(13, 35, closed="right"),
-    "35~60B": pd.Interval(35, 60, closed="right"),
-    "60B+": pd.Interval(60, 10000, closed="right"),
-}
+
+def filter_queries(query: str, filtered_df: pd.DataFrame):
+    """Added by Abishek"""
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+            )
+
+    return filtered_df
+
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_merges: bool, show_flagged: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -275,9 +141,15 @@ def filter_models(
     else:  # Show only still on the hub models
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
+    if not show_merges:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]
+
+    if not show_flagged:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
+
     type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df[df[AutoEvalColumn.precision.name].isin(precision_query)]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -286,6 +158,7 @@ def filter_models(
 
     return filtered_df
 
+leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False, False)
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -298,33 +171,21 @@ with demo:
                 with gr.Column():
                     with gr.Row():
                         search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model and press ENTER...",
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                             show_label=False,
                             elem_id="search-bar",
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
                             choices=[
-                                c
-                                for c in COLS
-                                if c
-                                not in [
-                                    AutoEvalColumn.dummy.name,
-                                    AutoEvalColumn.model.name,
-                                    AutoEvalColumn.model_type_symbol.name,
-                                    AutoEvalColumn.still_on_hub.name,
-                                ]
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden and not c.dummy
                             ],
                             value=[
-                                c
-                                for c in COLS_LITE
-                                if c
-                                not in [
-                                    AutoEvalColumn.dummy.name,
-                                    AutoEvalColumn.model.name,
-                                    AutoEvalColumn.model_type_symbol.name,
-                                    AutoEvalColumn.still_on_hub.name,
-                                ]
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
                             label="Select columns to show",
                             elem_id="column-select",
@@ -332,160 +193,132 @@ with demo:
                         )
                     with gr.Row():
                         deleted_models_visibility = gr.Checkbox(
-                            value=True, label="👀 Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    with gr.Box(elem_id="box-filter"):
-                        filter_columns_type = gr.CheckboxGroup(
-                            label="Model types",
-                            choices=[
-                                ModelType.PT.to_str(),
-                                # ModelType.FT.to_str(),
-                                ModelType.IFT.to_str(),
-                                ModelType.RL.to_str(),
-                            ],
-                            value=[
-                                ModelType.PT.to_str(),
-                                # ModelType.FT.to_str(),
-                                ModelType.IFT.to_str(),
-                                ModelType.RL.to_str(),
-                            ],
-                            interactive=True,
-                            elem_id="filter-columns-type",
+                            value=False, label="Show private/deleted models", interactive=True
                         )
-                        filter_columns_precision = gr.CheckboxGroup(
-                            label="Precision",
-                            choices=["torch.float16"], #, "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
-                            value=["torch.float16"], #, "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
-                            interactive=False,
-                            elem_id="filter-columns-precision",
+                        merged_models_visibility = gr.Checkbox(
+                            value=False, label="Show merges", interactive=True
                         )
-                        filter_columns_size = gr.CheckboxGroup(
-                            label="Model sizes",
-                            choices=list(NUMERIC_INTERVALS.keys()),
-                            value=list(NUMERIC_INTERVALS.keys()),
-                            interactive=True,
-                            elem_id="filter-columns-size",
+                        flagged_models_visibility = gr.Checkbox(
+                            value=False, label="Show flagged models", interactive=True
                         )
+                with gr.Column(min_width=320):
+                    #with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_precision = gr.CheckboxGroup(
+                        label="Precision",
+                        choices=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision],
+                        interactive=True,
+                        elem_id="filter-columns-precision",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
-                    [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
                     + [AutoEvalColumn.dummy.name]
                 ],
-                headers=[
-                    AutoEvalColumn.model_type_symbol.name,
-                    AutoEvalColumn.model.name,
-                ]
-                + shown_columns.value
-                + [AutoEvalColumn.dummy.name],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
-                max_rows=None,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
+                #column_widths=["2%", "33%"] 
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df,
+                value=original_df[COLS],
                 headers=COLS,
                 datatype=TYPES,
-                max_rows=None,
                 visible=False,
             )
             search_bar.submit(
                 update_table,
                 [
                     hidden_leaderboard_table_for_search,
-                    leaderboard_table,
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    merged_models_visibility,
+                    flagged_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
-            shown_columns.change(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    leaderboard_table,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-                queue=True,
-            )
-            filter_columns_type.change(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    leaderboard_table,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-                queue=True,
-            )
-            filter_columns_precision.change(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    leaderboard_table,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-                queue=True,
-            )
-            filter_columns_size.change(
+
+            # Define a hidden component that will trigger a reload only if a query parameter has be set
+            hidden_search_bar = gr.Textbox(value="", visible=False)
+            hidden_search_bar.change(
                 update_table,
                 [
                     hidden_leaderboard_table_for_search,
-                    leaderboard_table,
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    merged_models_visibility,
+                    flagged_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
-                queue=True,
             )
-            deleted_models_visibility.change(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
+            # Check query parameter once at startup and update search bar + hidden component
+            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
+            
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, merged_models_visibility, flagged_models_visibility]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_columns_type,
+                        filter_columns_precision,
+                        filter_columns_size,
+                        deleted_models_visibility,
+                        merged_models_visibility,
+                        flagged_models_visibility,
+                        search_bar,
+                    ],
                     leaderboard_table,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-                queue=True,
-            )
+                    queue=True,
+                )
+
+        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
+            with gr.Row():
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        [AutoEvalColumn.average.name],
+                        title="Average of Top Scores Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500) 
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        BENCHMARK_COLS,
+                        title="Top Scores Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500) 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
@@ -502,7 +335,7 @@ with demo:
                                 value=finished_eval_queue_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
-                                max_rows=5,
+                                row_count=5,
                             )
                     with gr.Accordion(
                         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
@@ -513,7 +346,7 @@ with demo:
                                 value=running_eval_queue_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
-                                max_rows=5,
+                                row_count=5,
                             )
 
                     with gr.Accordion(
@@ -525,7 +358,7 @@ with demo:
                                 value=pending_eval_queue_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
-                                max_rows=5,
+                                row_count=5,
                             )
                     with gr.Accordion(
                         f"❌ Failed Evaluations ({len(failed_eval_queue_df)})",
@@ -536,7 +369,7 @@ with demo:
                                 value=failed_eval_queue_df,
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
-                                max_rows=5,
+                                row_count=5,
                             )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
@@ -544,37 +377,26 @@ with demo:
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision", placeholder="main")
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
-                        choices=[
-                            ModelType.PT.to_str(" : "),
-                            # ModelType.FT.to_str(" : "),
-                            ModelType.IFT.to_str(" : "),
-                            ModelType.RL.to_str(" : "),
-                        ],
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
                         multiselect=False,
-                        value=None,
+                        value=ModelType.IFT.to_str(" : "),
                         interactive=True,
                     )
 
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=[
-                            "float16",
-                            # "bfloat16",
-                            # "8bit (LLM.int8)",
-                            # "4bit (QLoRA / FP4)",
-                            # "GPTQ"
-                        ],
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
                         label="Precision",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
-                        choices=["Original", "Delta", "Adapter"],
+                        choices=[i.value.name for i in WeightType],
                         label="Weights type",
                         multiselect=False,
                         value="Original",
@@ -603,20 +425,23 @@ with demo:
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
+                lines=20,
                 elem_id="citation-button",
-            ).style(show_copy_button=True)
-
+                show_copy_button=True,
+            )
     gr.HTML(BOTTOM_LOGO)
 
-    dummy = gr.Textbox(visible=False)
-    demo.load(
-        change_tab,
-        dummy,
-        tabs,
-        _js=get_window_url_params,
-    )
-
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(concurrency_count=40).launch()
+
+# Both launches the space and its CI
+configure_space_ci(
+    demo.queue(default_concurrency_limit=40),
+    trusted_authors=[],  # add manually trusted authors
+    private="True",  # ephemeral spaces will have same visibility as the main space. Otherwise, set to `True` or `False` explicitly.
+    variables={},  # We overwrite HF_HOME as tmp CI spaces will have no cache 
+    secrets=["HF_TOKEN", "H4_TOKEN"],  # which secret do I want to copy from the main space? Can be a `List[str]`.
+    hardware=None,  # "cpu-basic" by default. Otherwise set to "auto" to have same hardware as the main space or any valid string value.
+    storage=None,  # no storage by default. Otherwise set to "auto" to have same storage as the main space or any valid string value.
+).launch()

model_info_cache.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:337f1fb80e92327e7c7b130c03617439f7923e3f7c5383f5abb07e017ef9cae3
-size 715983

model_size_cache.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:64d63b51e6f5d6dd985b44ef6ddf513d9a7a138e734d77ae7382fd7a49a137ea
-size 20652

models_backlinks.py

@@ -1 +0,0 @@
-models = ['upstage/Llama-2-70b-instruct-v2', 'upstage/Llama-2-70b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-30b-instruct-2048', 'upstage/llama-30b-instruct', 'baseline']

package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "open_llm_leaderboard",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {}
+}

requirements.txt

@@ -1,71 +1,18 @@
-accelerate==0.23.0
-aiofiles==23.1.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-altair==4.2.2
-anyio==3.6.2
 APScheduler==3.10.1
-async-timeout==4.0.2
-attrs==23.1.0
-certifi==2022.12.7
-charset-normalizer==3.1.0
+black==23.11.0
 click==8.1.3
-contourpy==1.0.7
-cycler==0.11.0
-datasets==2.12.0
-entrypoints==0.4
-fastapi==0.95.1
-ffmpy==0.3.0
-filelock==3.11.0
-fonttools==4.39.3
-frozenlist==1.3.3
-fsspec==2023.4.0
-gradio==3.43.2
-gradio-client==0.5.0
-h11==0.14.0
-httpcore==0.17.0
-httpx==0.24.0
-huggingface-hub==0.16.4
-idna==3.4
-Jinja2==3.1.2
-jsonschema==4.17.3
-kiwisolver==1.4.4
-linkify-it-py==2.0.0
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
+datasets==2.14.5
+gradio==4.9.0
+gradio_client==0.7.2
+huggingface-hub>=0.18.0
 matplotlib==3.7.1
-mdit-py-plugins==0.3.3
-mdurl==0.1.2
-multidict==6.0.4
 numpy==1.24.2
-orjson==3.8.10
-packaging==23.1
 pandas==2.0.0
-Pillow==9.5.0
 plotly==5.14.1
-pyarrow==11.0.0
-pydantic==1.10.7
-pydub==0.25.1
-pyparsing==3.0.9
-pyrsistent==0.19.3
 python-dateutil==2.8.2
-python-multipart==0.0.6
-pytz==2023.3
-pytz-deprecation-shim==0.1.0.post0
-PyYAML==6.0
 requests==2.28.2
-semantic-version==2.10.0
-six==1.16.0
-sniffio==1.3.0
-starlette==0.26.1
-toolz==0.12.0
+sentencepiece
 tqdm==4.65.0
-transformers==4.34.0
-typing_extensions==4.5.0
-tzdata==2023.3
-tzlocal==4.3
-uc-micro-py==1.0.1
-urllib3==1.26.15
-uvicorn==0.21.1
-websockets==11.0.1
-yarl==1.8.2
+transformers==4.36.0
+tokenizers>=0.15.0
+gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/[email protected] # CI !!!

scripts/create_request_file.py

@@ -0,0 +1,107 @@
+import json
+import os
+import pprint
+import re
+from datetime import datetime, timezone
+
+import click
+from colorama import Fore
+from huggingface_hub import HfApi, snapshot_download
+
+EVAL_REQUESTS_PATH = "eval-queue"
+QUEUE_REPO = "open-ko-llm-leaderboard/requests"
+
+precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
+model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
+weight_types = ("Original", "Delta", "Adapter")
+
+
+def get_model_size(model_info, precision: str):
+    size_pattern =  re.compile(r"(\d+\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        try:
+            size_match = re.search(size_pattern, model_info.modelId.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+
+def main():
+    api = HfApi()
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
+
+    model_name = click.prompt("Enter model name")
+    revision = click.prompt("Enter revision", default="main")
+    precision = click.prompt("Enter precision", default="float16", type=click.Choice(precisions))
+    model_type = click.prompt("Enter model type", type=click.Choice(model_types))
+    weight_type = click.prompt("Enter weight type", default="Original", type=click.Choice(weight_types))
+    base_model = click.prompt("Enter base model", default="")
+    status = click.prompt("Enter status", default="FINISHED")
+
+    try:
+        model_info = api.model_info(repo_id=model_name, revision=revision)
+    except Exception as e:
+        print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}")
+        return 1
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        license = "?"
+
+    eval_entry = {
+        "model": model_name,
+        "base_model": base_model,
+        "revision": revision,
+        "private": False,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": status,
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+    }
+
+    user_name = ""
+    model_path = model_name
+    if "/" in model_name:
+        user_name = model_name.split("/")[0]
+        model_path = model_name.split("/")[1]
+
+    pprint.pprint(eval_entry)
+
+    if click.confirm("Do you want to continue? This request file will be pushed to the hub"):
+        click.echo("continuing...")
+
+        out_dir = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(out_dir, exist_ok=True)
+        out_path = f"{out_dir}/{model_path}_eval_request_{False}_{precision}_{weight_type}.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+
+        api.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name} to eval queue",
+        )
+    else:
+        click.echo("aborting...")
+
+
+if __name__ == "__main__":
+    main()

scripts/update_request_files.py

@@ -0,0 +1,82 @@
+import json
+import os
+import glob
+import pprint
+import re
+from datetime import datetime, timezone
+
+import click
+from colorama import Fore
+from huggingface_hub import HfApi, snapshot_download
+from huggingface_hub.hf_api import ModelInfo
+
+API = HfApi()
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError ):
+        try:
+            size_match = re.search(size_pattern, model_info.modelId.split("/")[-1].lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.split("/")[-1].lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+
+def update_request_files(requests_path):
+    request_files = os.path.join(
+        requests_path, "*/*.json"
+    )
+    request_files = glob.glob(request_files)
+
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            new_req_content = add_model_info(req_content)
+        
+        # if new content is different, update the file
+        if new_req_content != req_content:
+            with open(tmp_request_file, "w") as f:
+                f.write(json.dumps(new_req_content, indent=4))
+
+def add_model_info(entry):
+
+    model = entry["model"]
+    revision = entry["revision"]
+
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        print(f"Could not get model information for {model} revision {revision}")
+        return entry
+
+    new_entry = entry.copy()
+
+    model_size = get_model_size(model_info=model_info, precision='float16')
+    new_entry["params"] = model_size
+
+    new_entry["likes"] = model_info.likes
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+        new_entry["license"] = license
+    except Exception:
+        print(f"No license for {model} revision {revision}")
+
+    print(json.dumps(new_entry, indent=4))
+    return new_entry
+
+
+if __name__ == "__main__":
+    # update_request_files("/Users/sean/workspace/leaderboard/leaderboard-test-requests")
+    update_request_files("/Volumes/Data-case-sensitive/requests")
+    

src/assets/hardcoded_evals.py

@@ -1,14 +0,0 @@
-from src.display_models.utils import AutoEvalColumn, model_hyperlink
-
-baseline = {
-    AutoEvalColumn.model.name: "<p>Baseline</p>",
-    AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 25.0,
-    AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name: 25.0,
-    AutoEvalColumn.mmlu.name: 25.0,
-    AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.dummy.name: "baseline",
-    AutoEvalColumn.model_type.name: "",
-}

src/{assets/text_content.py → display/about.py}

@@ -1,4 +1,5 @@
-from src.display_models.model_metadata_type import ModelType
+from src.display.utils import ModelType
+
 
 TITLE = """<img src="https://upstage-open-ko-llm-leaderboard-logos.s3.ap-northeast-2.amazonaws.com/header_logo.png" style="width:30%;display:block;margin-left:auto;margin-right:auto">"""
 BOTTOM_LOGO = """<img src="https://upstage-open-ko-llm-leaderboard-logos.s3.ap-northeast-2.amazonaws.com/footer_logo_1.png" style="width:50%;display:block;margin-left:auto;margin-right:auto">"""
@@ -20,7 +21,6 @@ While outstanding LLM models are being released competitively, most of them are
 
 ## Icons
 {ModelType.PT.to_str(" : ")} model
-{ModelType.FT.to_str(" : ")} model
 {ModelType.IFT.to_str(" : ")} model
 {ModelType.RL.to_str(" : ")} model
 If there is no icon, it indicates that there is insufficient information about the model.
@@ -52,6 +52,11 @@ GPUs are provided by __[KT](https://cloud.kt.com/)__ for the evaluations.
 If you still have questions, you can check our FAQ [here](https://huggingface.co/spaces/upstage/open-ko-llm-leaderboard/discussions/1)!
 """
 
+
+FAQ_TEXT = """
+"""
+
+
 EVALUATION_QUEUE_TEXT = f"""
 # Evaluation Queue for the 🚀 Open Ko-LLM Leaderboard
 Models added here will be automatically evaluated on the KT GPU cluster.

src/{assets → display}/css_html_js.py

@@ -1,5 +1,24 @@
 custom_css = """
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
 
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+/* Full width space */
+.gradio-container {
+  max-width: 95%!important;
+}
+
+/* Text style and margins */
 .markdown-text {
     font-size: 16px !important;
 }
@@ -21,14 +40,6 @@ custom_css = """
     transform: scale(1.3);
 }
 
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
 #search-bar-table-box > div:first-child {
     background: none;
     border: none;
@@ -38,36 +49,11 @@ custom_css = """
     padding: 0px;
 }
 
-/* Hides the final AutoEvalColumn */
-#llm-benchmark-tab-table table td:last-child,
-#llm-benchmark-tab-table table th:last-child {
-    display: none;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
 .tab-buttons button {
     font-size: 20px;
 }
 
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
+/* Filters style */
 #filter_type{
     border: 0;
     padding-left: 0;

src/display/formatting.py

@@ -0,0 +1,40 @@
+import os
+from datetime import datetime, timezone
+
+from huggingface_hub import HfApi
+from huggingface_hub.hf_api import ModelInfo
+
+
+API = HfApi()
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    details_model_name = model_name.replace("/", "__")
+    details_link = f"https://huggingface.co/datasets/open-ko-llm-leaderboard/details_{details_model_name}"
+
+    return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,151 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class Tasks(Enum):
+    arc = Task("ko_arc_challenge", "acc_norm", "Ko-ARC")
+    hellaswag = Task("ko_hellaswag", "acc_norm", "Ko-HellaSwag")
+    mmlu = Task("ko_mmlu", "acc", "Ko-MMLU")
+    truthfulqa = Task("ko_truthfulqa_mc", "mc2", "Ko-TruthfulQA")
+    commongen_v2 = Task("ko_commongen_v2", "acc_norm", "Ko-CommonGen V2")
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+# Define the human baselines
+human_baseline_row = {
+    AutoEvalColumn.model.name: "<p>Human performance</p>",
+}
+
+@dataclass
+class ModelDetails:
+    name: str
+    symbol: str = "" # emoji, only for the model type
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    # FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        # if "fine-tuned" in type or "🔶" in type:
+        #     return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    # bfloat16 = ModelDetails("bfloat16")
+    # qt_8bit = ModelDetails("8bit")
+    # qt_4bit = ModelDetails("4bit")
+    # qt_GPTQ = ModelDetails("GPTQ")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        if precision in ["8bit"]:
+            return Precision.qt_8bit
+        if precision in ["4bit"]:
+            return Precision.qt_4bit
+        if precision in ["GPTQ", "None"]:
+            return Precision.qt_GPTQ
+        return Precision.Unknown
+        
+
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+
+NUMERIC_INTERVALS = {
+    "Unknown": pd.Interval(-1, 0, closed="right"),
+    "0~3B": pd.Interval(0, 3, closed="right"),
+    "3~7B": pd.Interval(3, 7.3, closed="right"),
+    "7~13B": pd.Interval(7.3, 13, closed="right"),
+    "13~35B": pd.Interval(13, 35, closed="right"),
+    "35~60B": pd.Interval(35, 60, closed="right"),
+    "60B+": pd.Interval(60, 10000, closed="right"),
+}

src/display_models/get_model_metadata.py

@@ -1,167 +0,0 @@
-import glob
-import json
-import os
-import re
-import pickle
-from typing import List
-
-import huggingface_hub
-from huggingface_hub import HfApi
-from tqdm import tqdm
-from transformers import AutoModel, AutoConfig
-from accelerate import init_empty_weights
-
-from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
-from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
-from src.display_models.utils import AutoEvalColumn, model_hyperlink
-
-api = HfApi(token=os.environ.get("H4_TOKEN", None))
-
-
-def get_model_infos_from_hub(leaderboard_data: List[dict]):
-    # load cache from disk
-    try:
-        with open("model_info_cache.pkl", "rb") as f:
-            model_info_cache = pickle.load(f)
-    except (EOFError, FileNotFoundError):
-        model_info_cache = {}
-    try:
-        with open("model_size_cache.pkl", "rb") as f:
-            model_size_cache = pickle.load(f)
-    except (EOFError, FileNotFoundError):
-        model_size_cache = {}
-
-    for model_data in tqdm(leaderboard_data):
-        model_name = model_data["model_name_for_query"]
-
-        if model_name in model_info_cache:
-            model_info = model_info_cache[model_name]
-        else:
-            try:
-                model_info = api.model_info(model_name)
-                model_info_cache[model_name] = model_info
-            except huggingface_hub.utils._errors.RepositoryNotFoundError:
-                print("Repo not found!", model_name)
-                model_data[AutoEvalColumn.license.name] = None
-                model_data[AutoEvalColumn.likes.name] = None
-                if model_name not in model_size_cache:
-                    model_size_cache[model_name] = get_model_size(model_name, None)
-                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
-
-        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
-        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
-        if model_name not in model_size_cache:
-            model_size_cache[model_name] = get_model_size(model_name, model_info)
-        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
-    
-    # save cache to disk in pickle format
-    with open("model_info_cache.pkl", "wb") as f:
-        pickle.dump(model_info_cache, f)
-    with open("model_size_cache.pkl", "wb") as f:
-        pickle.dump(model_size_cache, f)
-
-
-def get_model_license(model_info):
-    try:
-        return model_info.cardData["license"]
-    except Exception:
-        return "?"
-
-
-def get_model_likes(model_info):
-    return model_info.likes
-
-
-size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
-
-
-def get_model_size(model_name, model_info):
-    # In billions
-    try:
-        return round(model_info.safetensors["total"] / 1e9, 3)
-    except AttributeError:
-        try:
-            config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
-            with init_empty_weights():
-                model = AutoModel.from_config(config, trust_remote_code=False)
-            return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
-        except (EnvironmentError, ValueError): # model config not found, likely private  
-            try:
-                size_match = re.search(size_pattern, model_name.lower())
-                size = size_match.group(0)
-                return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
-            except AttributeError:
-                return 0
-
-
-def get_model_type(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        request_files = os.path.join(
-            "eval-queue",
-            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
-        )
-        request_files = glob.glob(request_files)
-
-        # Select correct request file (precision)
-        request_file = ""
-        if len(request_files) == 1:
-            request_file = request_files[0]
-        elif len(request_files) > 1:
-            request_files = sorted(request_files, reverse=True)
-            for tmp_request_file in request_files:
-                with open(tmp_request_file, "r") as f:
-                    req_content = json.load(f)
-                    if (
-                        req_content["status"] == "FINISHED"
-                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
-                    ):
-                        request_file = tmp_request_file
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            model_type = model_type_from_str(request["model_type"])
-            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
-            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
-        except Exception:
-            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
-                    model_data["model_name_for_query"]
-                ].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
-                    model_data["model_name_for_query"]
-                ].value.symbol  # + ("🔺" if is_delta else "")
-            else:
-                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
-
-
-def flag_models(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        if model_data["model_name_for_query"] in FLAGGED_MODELS:
-            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
-            issue_link = model_hyperlink(
-                FLAGGED_MODELS[model_data["model_name_for_query"]],
-                f"See discussion #{issue_num}",
-            )
-            model_data[
-                AutoEvalColumn.model.name
-            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
-
-
-def remove_forbidden_models(leaderboard_data: List[dict]):
-    indices_to_remove = []
-    for ix, model in enumerate(leaderboard_data):
-        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
-            indices_to_remove.append(ix)
-
-    for ix in reversed(indices_to_remove):
-        leaderboard_data.pop(ix)
-    return leaderboard_data
-
-
-def apply_metadata(leaderboard_data: List[dict]):
-    leaderboard_data = remove_forbidden_models(leaderboard_data)
-    get_model_type(leaderboard_data)
-    get_model_infos_from_hub(leaderboard_data)
-    flag_models(leaderboard_data)

src/display_models/model_metadata_flags.py

@@ -1,8 +0,0 @@
-# Models which have been flagged by users as being problematic for a reason or another
-# (Model name to forum discussion link)
-FLAGGED_MODELS = {
-}
-
-# Models which have been requested by orgs to not be submitted on the leaderboard
-DO_NOT_SUBMIT_MODELS = [
-]

src/display_models/model_metadata_type.py

@@ -1,553 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Dict
-
-
-@dataclass
-class ModelInfo:
-    name: str
-    symbol: str  # emoji
-
-
-class ModelType(Enum):
-    PT = ModelInfo(name="pretrained", symbol="🟢")
-    FT = ModelInfo(name="fine-tuned", symbol="🔶")
-    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
-    RL = ModelInfo(name="RL-tuned", symbol="🟦")
-    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-
-MODEL_TYPE_METADATA: Dict[str, ModelType] = {
-    "tiiuae/falcon-180B": ModelType.PT,
-    "Qwen/Qwen-7B": ModelType.PT,
-    "Qwen/Qwen-7B-Chat": ModelType.RL,
-    "notstoic/PygmalionCoT-7b": ModelType.IFT,
-    "aisquared/dlite-v1-355m": ModelType.IFT,
-    "aisquared/dlite-v1-1_5b": ModelType.IFT,
-    "aisquared/dlite-v1-774m": ModelType.IFT,
-    "aisquared/dlite-v1-124m": ModelType.IFT,
-    "aisquared/chopt-2_7b": ModelType.IFT,
-    "aisquared/dlite-v2-124m": ModelType.IFT,
-    "aisquared/dlite-v2-774m": ModelType.IFT,
-    "aisquared/dlite-v2-1_5b": ModelType.IFT,
-    "aisquared/chopt-1_3b": ModelType.IFT,
-    "aisquared/dlite-v2-355m": ModelType.IFT,
-    "augtoma/qCammel-13": ModelType.IFT,
-    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload": ModelType.IFT,
-    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload": ModelType.IFT,
-    "TheBloke/alpaca-lora-65B-HF": ModelType.FT,
-    "TheBloke/tulu-7B-fp16": ModelType.IFT,
-    "TheBloke/guanaco-7B-HF": ModelType.FT,
-    "TheBloke/koala-7B-HF": ModelType.FT,
-    "TheBloke/wizardLM-7B-HF": ModelType.IFT,
-    "TheBloke/airoboros-13B-HF": ModelType.IFT,
-    "TheBloke/koala-13B-HF": ModelType.FT,
-    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF": ModelType.FT,
-    "TheBloke/dromedary-65b-lora-HF": ModelType.IFT,
-    "TheBloke/wizardLM-13B-1.0-fp16": ModelType.IFT,
-    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16": ModelType.FT,
-    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16": ModelType.FT,
-    "TheBloke/wizard-vicuna-13B-HF": ModelType.IFT,
-    "TheBloke/UltraLM-13B-fp16": ModelType.IFT,
-    "TheBloke/OpenAssistant-FT-7-Llama-30B-HF": ModelType.FT,
-    "TheBloke/vicuna-13B-1.1-HF": ModelType.IFT,
-    "TheBloke/guanaco-13B-HF": ModelType.FT,
-    "TheBloke/guanaco-65B-HF": ModelType.FT,
-    "TheBloke/airoboros-7b-gpt4-fp16": ModelType.IFT,
-    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Llama-2-13B-fp16": ModelType.PT,
-    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16": ModelType.FT,
-    "TheBloke/landmark-attention-llama7b-fp16": ModelType.IFT,
-    "TheBloke/Planner-7B-fp16": ModelType.IFT,
-    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.FT,
-    "TheBloke/gpt4-alpaca-lora-13B-HF": ModelType.IFT,
-    "TheBloke/gpt4-x-vicuna-13B-HF": ModelType.IFT,
-    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF": ModelType.IFT,
-    "TheBloke/tulu-13B-fp16": ModelType.IFT,
-    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16": ModelType.IFT,
-    "TheBloke/Llama-2-70B-fp16": ModelType.IFT,
-    "TheBloke/WizardLM-30B-fp16": ModelType.IFT,
-    "TheBloke/robin-13B-v2-fp16": ModelType.FT,
-    "TheBloke/robin-33B-v2-fp16": ModelType.FT,
-    "TheBloke/Vicuna-13B-CoT-fp16": ModelType.IFT,
-    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16": ModelType.FT,
-    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16": ModelType.FT,
-    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16": ModelType.IFT,
-    "jphme/orca_mini_v2_ger_7b": ModelType.IFT,
-    "Ejafa/vicuna_7B_vanilla_1.1": ModelType.FT,
-    "kevinpro/Vicuna-13B-CoT": ModelType.IFT,
-    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml": ModelType.FT,
-    "AlekseyKorshuk/chatml-pyg-v1": ModelType.FT,
-    "concedo/Vicuzard-30B-Uncensored": ModelType.FT,
-    "concedo/OPT-19M-ChatSalad": ModelType.FT,
-    "concedo/Pythia-70M-ChatSalad": ModelType.FT,
-    "digitous/13B-HyperMantis": ModelType.IFT,
-    "digitous/Adventien-GPTJ": ModelType.FT,
-    "digitous/Alpacino13b": ModelType.IFT,
-    "digitous/GPT-R": ModelType.IFT,
-    "digitous/Javelin-R": ModelType.IFT,
-    "digitous/Javalion-GPTJ": ModelType.IFT,
-    "digitous/Javalion-R": ModelType.IFT,
-    "digitous/Skegma-GPTJ": ModelType.FT,
-    "digitous/Alpacino30b": ModelType.IFT,
-    "digitous/Janin-GPTJ": ModelType.FT,
-    "digitous/Janin-R": ModelType.FT,
-    "digitous/Javelin-GPTJ": ModelType.FT,
-    "SaylorTwift/gpt2_test": ModelType.PT,
-    "anton-l/gpt-j-tiny-random": ModelType.FT,
-    "Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca": ModelType.FT,
-    "Lazycuber/pyg-instruct-wizardlm": ModelType.FT,
-    "Lazycuber/Janemalion-6B": ModelType.FT,
-    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.FT,
-    "IDEA-CCNL/Ziya-LLaMA-13B-v1": ModelType.IFT,
-    "dsvv-cair/alpaca-cleaned-llama-30b-bf16": ModelType.FT,
-    "gpt2-medium": ModelType.PT,
-    "camel-ai/CAMEL-13B-Combined-Data": ModelType.IFT,
-    "camel-ai/CAMEL-13B-Role-Playing-Data": ModelType.FT,
-    "camel-ai/CAMEL-33B-Combined-Data": ModelType.IFT,
-    "PygmalionAI/pygmalion-6b": ModelType.FT,
-    "PygmalionAI/metharme-1.3b": ModelType.IFT,
-    "PygmalionAI/pygmalion-1.3b": ModelType.FT,
-    "PygmalionAI/pygmalion-350m": ModelType.FT,
-    "PygmalionAI/pygmalion-2.7b": ModelType.FT,
-    "medalpaca/medalpaca-7b": ModelType.FT,
-    "lilloukas/Platypus-30B": ModelType.IFT,
-    "lilloukas/GPlatty-30B": ModelType.FT,
-    "mncai/chatdoctor": ModelType.FT,
-    "chaoyi-wu/MedLLaMA_13B": ModelType.FT,
-    "LoupGarou/WizardCoder-Guanaco-15B-V1.0": ModelType.IFT,
-    "LoupGarou/WizardCoder-Guanaco-15B-V1.1": ModelType.FT,
-    "hakurei/instruct-12b": ModelType.IFT,
-    "hakurei/lotus-12B": ModelType.FT,
-    "shibing624/chinese-llama-plus-13b-hf": ModelType.IFT,
-    "shibing624/chinese-alpaca-plus-7b-hf": ModelType.IFT,
-    "shibing624/chinese-alpaca-plus-13b-hf": ModelType.IFT,
-    "mosaicml/mpt-7b-instruct": ModelType.IFT,
-    "mosaicml/mpt-30b-chat": ModelType.IFT,
-    "mosaicml/mpt-7b-storywriter": ModelType.FT,
-    "mosaicml/mpt-30b-instruct": ModelType.IFT,
-    "mosaicml/mpt-7b-chat": ModelType.IFT,
-    "mosaicml/mpt-30b": ModelType.PT,
-    "Corianas/111m": ModelType.IFT,
-    "Corianas/Quokka_1.3b": ModelType.IFT,
-    "Corianas/256_5epoch": ModelType.FT,
-    "Corianas/Quokka_256m": ModelType.IFT,
-    "Corianas/Quokka_590m": ModelType.IFT,
-    "Corianas/gpt-j-6B-Dolly": ModelType.FT,
-    "Corianas/Quokka_2.7b": ModelType.IFT,
-    "cyberagent/open-calm-7b": ModelType.FT,
-    "Aspik101/Nous-Hermes-13b-pl-lora_unload": ModelType.IFT,
-    "THUDM/chatglm2-6b": ModelType.IFT,
-    "MetaIX/GPT4-X-Alpasta-30b": ModelType.IFT,
-    "NYTK/PULI-GPTrio": ModelType.PT,
-    "EleutherAI/pythia-1.3b": ModelType.PT,
-    "EleutherAI/pythia-2.8b-deduped": ModelType.PT,
-    "EleutherAI/gpt-neo-125m": ModelType.PT,
-    "EleutherAI/pythia-160m": ModelType.PT,
-    "EleutherAI/gpt-neo-2.7B": ModelType.PT,
-    "EleutherAI/pythia-1b-deduped": ModelType.PT,
-    "EleutherAI/pythia-6.7b": ModelType.PT,
-    "EleutherAI/pythia-70m-deduped": ModelType.PT,
-    "EleutherAI/gpt-neox-20b": ModelType.PT,
-    "EleutherAI/pythia-1.4b-deduped": ModelType.PT,
-    "EleutherAI/pythia-2.7b": ModelType.PT,
-    "EleutherAI/pythia-6.9b-deduped": ModelType.PT,
-    "EleutherAI/pythia-70m": ModelType.PT,
-    "EleutherAI/gpt-j-6b": ModelType.PT,
-    "EleutherAI/pythia-12b-deduped": ModelType.PT,
-    "EleutherAI/gpt-neo-1.3B": ModelType.PT,
-    "EleutherAI/pythia-410m-deduped": ModelType.PT,
-    "EleutherAI/pythia-160m-deduped": ModelType.PT,
-    "EleutherAI/polyglot-ko-12.8b": ModelType.PT,
-    "EleutherAI/pythia-12b": ModelType.PT,
-    "roneneldan/TinyStories-33M": ModelType.PT,
-    "roneneldan/TinyStories-28M": ModelType.PT,
-    "roneneldan/TinyStories-1M": ModelType.PT,
-    "roneneldan/TinyStories-8M": ModelType.PT,
-    "roneneldan/TinyStories-3M": ModelType.PT,
-    "jerryjalapeno/nart-100k-7b": ModelType.FT,
-    "lmsys/vicuna-13b-v1.3": ModelType.IFT,
-    "lmsys/vicuna-7b-v1.3": ModelType.IFT,
-    "lmsys/vicuna-13b-v1.1": ModelType.IFT,
-    "lmsys/vicuna-13b-delta-v1.1": ModelType.IFT,
-    "lmsys/vicuna-7b-delta-v1.1": ModelType.IFT,
-    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.FT,
-    "haonan-li/bactrian-x-llama-13b-merged": ModelType.IFT,
-    "Gryphe/MythoLogic-13b": ModelType.IFT,
-    "Gryphe/MythoBoros-13b": ModelType.IFT,
-    "pillowtalks-ai/delta13b": ModelType.FT,
-    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard": ModelType.FT,
-    "bigscience/bloom-7b1": ModelType.PT,
-    "bigcode/tiny_starcoder_py": ModelType.PT,
-    "bigcode/starcoderplus": ModelType.FT,
-    "bigcode/gpt_bigcode-santacoder": ModelType.PT,
-    "bigcode/starcoder": ModelType.PT,
-    "Open-Orca/OpenOrca-Preview1-13B": ModelType.IFT,
-    "microsoft/DialoGPT-large": ModelType.FT,
-    "microsoft/DialoGPT-small": ModelType.FT,
-    "microsoft/DialoGPT-medium": ModelType.FT,
-    "microsoft/CodeGPT-small-py": ModelType.FT,
-    "Tincando/fiction_story_generator": ModelType.FT,
-    "Pirr/pythia-13b-deduped-green_devil": ModelType.FT,
-    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.FT,
-    "Aeala/GPT4-x-AlpacaDente-30b": ModelType.FT,
-    "Aeala/GPT4-x-Alpasta-13b": ModelType.FT,
-    "Aeala/VicUnlocked-alpaca-30b": ModelType.IFT,
-    "Tap-M/Luna-AI-Llama2-Uncensored": ModelType.FT,
-    "illuin/test-custom-llama": ModelType.FT,
-    "dvruette/oasst-llama-13b-2-epochs": ModelType.FT,
-    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.FT,
-    "dvruette/llama-13b-pretrained-dropout": ModelType.PT,
-    "dvruette/llama-13b-pretrained": ModelType.PT,
-    "dvruette/llama-13b-pretrained-sft-epoch-1": ModelType.FT,
-    "dvruette/llama-13b-pretrained-sft-do2": ModelType.FT,
-    "dvruette/oasst-gpt-neox-20b-3000-steps": ModelType.FT,
-    "dvruette/oasst-pythia-12b-pretrained-sft": ModelType.FT,
-    "dvruette/oasst-pythia-6.9b-4000-steps": ModelType.FT,
-    "dvruette/gpt-neox-20b-full-precision": ModelType.FT,
-    "dvruette/oasst-llama-13b-1000-steps": ModelType.FT,
-    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
-    "openlm-research/open_llama_7b": ModelType.PT,
-    "openlm-research/open_llama_7b_v2": ModelType.PT,
-    "openlm-research/open_llama_3b": ModelType.PT,
-    "openlm-research/open_llama_13b": ModelType.PT,
-    "openlm-research/open_llama_3b_v2": ModelType.PT,
-    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.IFT,
-    "GeorgiaTechResearchInstitute/galpaca-30b": ModelType.IFT,
-    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct": ModelType.IFT,
-    "databricks/dolly-v2-7b": ModelType.IFT,
-    "databricks/dolly-v2-3b": ModelType.IFT,
-    "databricks/dolly-v2-12b": ModelType.IFT,
-    "Rachneet/gpt2-xl-alpaca": ModelType.FT,
-    "Locutusque/gpt2-conversational-or-qa": ModelType.FT,
-    "psyche/kogpt": ModelType.FT,
-    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.IFT,
-    "Mikael110/llama-2-7b-guanaco-fp16": ModelType.FT,
-    "Mikael110/llama-2-13b-guanaco-fp16": ModelType.FT,
-    "Fredithefish/CrimsonPajama": ModelType.IFT,
-    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K": ModelType.FT,
-    "Fredithefish/ScarletPajama-3B-HF": ModelType.FT,
-    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4": ModelType.IFT,
-    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1": ModelType.IFT,
-    "eachadea/vicuna-13b-1.1": ModelType.FT,
-    "eachadea/vicuna-7b-1.1": ModelType.FT,
-    "eachadea/vicuna-13b": ModelType.FT,
-    "openaccess-ai-collective/wizard-mega-13b": ModelType.IFT,
-    "openaccess-ai-collective/manticore-13b": ModelType.IFT,
-    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.IFT,
-    "openaccess-ai-collective/minotaur-13b": ModelType.IFT,
-    "openaccess-ai-collective/minotaur-13b-fixed": ModelType.IFT,
-    "openaccess-ai-collective/hippogriff-30b-chat": ModelType.IFT,
-    "openaccess-ai-collective/manticore-13b-chat-pyg": ModelType.IFT,
-    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.IFT,
-    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.IFT,
-    "euclaise/gpt-neox-122m-minipile-digits": ModelType.FT,
-    "stabilityai/StableBeluga1-Delta": ModelType.IFT,
-    "stabilityai/stablelm-tuned-alpha-7b": ModelType.IFT,
-    "stabilityai/StableBeluga2": ModelType.IFT,
-    "stabilityai/StableBeluga-13B": ModelType.IFT,
-    "stabilityai/StableBeluga-7B": ModelType.IFT,
-    "stabilityai/stablelm-base-alpha-7b": ModelType.PT,
-    "stabilityai/stablelm-base-alpha-3b": ModelType.PT,
-    "stabilityai/stablelm-tuned-alpha-3b": ModelType.IFT,
-    "alibidaran/medical_transcription_generator": ModelType.FT,
-    "CalderaAI/30B-Lazarus": ModelType.IFT,
-    "CalderaAI/13B-BlueMethod": ModelType.IFT,
-    "CalderaAI/13B-Ouroboros": ModelType.IFT,
-    "KoboldAI/OPT-13B-Erebus": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Janeway": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Shinen": ModelType.FT,
-    "KoboldAI/fairseq-dense-2.7B": ModelType.PT,
-    "KoboldAI/OPT-6B-nerys-v2": ModelType.FT,
-    "KoboldAI/GPT-NeoX-20B-Skein": ModelType.FT,
-    "KoboldAI/PPO_Pygway-6b-Mix": ModelType.FT,
-    "KoboldAI/fairseq-dense-6.7B": ModelType.PT,
-    "KoboldAI/fairseq-dense-125M": ModelType.PT,
-    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Erebus": ModelType.FT,
-    "KoboldAI/OPT-350M-Nerys-v2": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Nerys-v2": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/OPT-13B-Nerys-v2": ModelType.FT,
-    "KoboldAI/GPT-NeoX-20B-Erebus": ModelType.FT,
-    "KoboldAI/OPT-6.7B-Erebus": ModelType.FT,
-    "KoboldAI/fairseq-dense-355M": ModelType.PT,
-    "KoboldAI/OPT-6.7B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Adventure": ModelType.FT,
-    "KoboldAI/OPT-350M-Erebus": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Skein": ModelType.FT,
-    "KoboldAI/OPT-30B-Erebus": ModelType.FT,
-    "klosax/pythia-160m-deduped-step92k-193bt": ModelType.PT,
-    "klosax/open_llama_3b_350bt_preview": ModelType.PT,
-    "klosax/openllama-3b-350bt": ModelType.PT,
-    "klosax/pythia-70m-deduped-step44k-92bt": ModelType.PT,
-    "klosax/open_llama_13b_600bt_preview": ModelType.PT,
-    "klosax/open_llama_7b_400bt_preview": ModelType.PT,
-    "kfkas/Llama-2-ko-7b-Chat": ModelType.IFT,
-    "WeOpenML/Alpaca-7B-v1": ModelType.IFT,
-    "WeOpenML/PandaLM-Alpaca-7B-v1": ModelType.IFT,
-    "TFLai/gpt2-turkish-uncased": ModelType.FT,
-    "ehartford/WizardLM-13B-Uncensored": ModelType.IFT,
-    "ehartford/dolphin-llama-13b": ModelType.IFT,
-    "ehartford/Wizard-Vicuna-30B-Uncensored": ModelType.FT,
-    "ehartford/WizardLM-30B-Uncensored": ModelType.IFT,
-    "ehartford/Wizard-Vicuna-13B-Uncensored": ModelType.FT,
-    "ehartford/WizardLM-7B-Uncensored": ModelType.IFT,
-    "ehartford/based-30b": ModelType.FT,
-    "ehartford/Wizard-Vicuna-7B-Uncensored": ModelType.FT,
-    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.FT,
-    "wahaha1987/llama_13b_sharegpt94k_fastchat": ModelType.FT,
-    "OpenAssistant/oasst-sft-1-pythia-12b": ModelType.FT,
-    "OpenAssistant/stablelm-7b-sft-v7-epoch-3": ModelType.IFT,
-    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.FT,
-    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps": ModelType.IFT,
-    "OpenAssistant/pythia-12b-sft-v8-7k-steps": ModelType.IFT,
-    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps": ModelType.IFT,
-    "OpenAssistant/llama2-13b-orca-8k-3319": ModelType.IFT,
-    "junelee/wizard-vicuna-13b": ModelType.FT,
-    "BreadAi/gpt-YA-1-1_160M": ModelType.PT,
-    "BreadAi/MuseCan": ModelType.PT,
-    "BreadAi/MusePy-1-2": ModelType.PT,
-    "BreadAi/DiscordPy": ModelType.PT,
-    "BreadAi/PM_modelV2": ModelType.PT,
-    "BreadAi/gpt-Youtube": ModelType.PT,
-    "BreadAi/StoryPy": ModelType.FT,
-    "julianweng/Llama-2-7b-chat-orcah": ModelType.FT,
-    "AGI-inc/lora_moe_7b_baseline": ModelType.FT,
-    "AGI-inc/lora_moe_7b": ModelType.FT,
-    "togethercomputer/GPT-NeoXT-Chat-Base-20B": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-7B-Base": ModelType.PT,
-    "togethercomputer/RedPajama-INCITE-7B-Instruct": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
-    "togethercomputer/Pythia-Chat-Base-7B": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
-    "togethercomputer/GPT-JT-6B-v1": ModelType.IFT,
-    "togethercomputer/GPT-JT-6B-v0": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-7B-Chat": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1": ModelType.IFT,
-    "Writer/camel-5b-hf": ModelType.IFT,
-    "Writer/palmyra-base": ModelType.PT,
-    "MBZUAI/LaMini-GPT-1.5B": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-111m": ModelType.IFT,
-    "MBZUAI/lamini-neo-1.3b": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-1.3b": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-256m": ModelType.IFT,
-    "MBZUAI/LaMini-GPT-124M": ModelType.IFT,
-    "MBZUAI/lamini-neo-125m": ModelType.IFT,
-    "TehVenom/DiffMerge-DollyGPT-Pygmalion": ModelType.FT,
-    "TehVenom/PPO_Shygmalion-6b": ModelType.FT,
-    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2": ModelType.FT,
-    "TehVenom/Pygmalion_AlpacaLora-7b": ModelType.FT,
-    "TehVenom/PPO_Pygway-V8p4_Dev-6b": ModelType.FT,
-    "TehVenom/Dolly_Malion-6b": ModelType.FT,
-    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b": ModelType.FT,
-    "TehVenom/ChanMalion": ModelType.FT,
-    "TehVenom/GPT-J-Pyg_PPO-6B": ModelType.IFT,
-    "TehVenom/Pygmalion-13b-Merged": ModelType.FT,
-    "TehVenom/Metharme-13b-Merged": ModelType.IFT,
-    "TehVenom/Dolly_Shygmalion-6b": ModelType.FT,
-    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4": ModelType.IFT,
-    "georgesung/llama2_7b_chat_uncensored": ModelType.FT,
-    "vicgalle/gpt2-alpaca": ModelType.IFT,
-    "vicgalle/alpaca-7b": ModelType.FT,
-    "vicgalle/gpt2-alpaca-gpt4": ModelType.IFT,
-    "facebook/opt-350m": ModelType.PT,
-    "facebook/opt-125m": ModelType.PT,
-    "facebook/xglm-4.5B": ModelType.PT,
-    "facebook/opt-2.7b": ModelType.PT,
-    "facebook/opt-6.7b": ModelType.PT,
-    "facebook/galactica-30b": ModelType.PT,
-    "facebook/opt-13b": ModelType.PT,
-    "facebook/opt-66b": ModelType.PT,
-    "facebook/xglm-7.5B": ModelType.PT,
-    "facebook/xglm-564M": ModelType.PT,
-    "facebook/opt-30b": ModelType.PT,
-    "golaxy/gogpt-7b": ModelType.FT,
-    "golaxy/gogpt2-7b": ModelType.FT,
-    "golaxy/gogpt-7b-bloom": ModelType.FT,
-    "golaxy/gogpt-3b-bloom": ModelType.FT,
-    "psmathur/orca_mini_v2_7b": ModelType.IFT,
-    "psmathur/orca_mini_7b": ModelType.IFT,
-    "psmathur/orca_mini_3b": ModelType.IFT,
-    "psmathur/orca_mini_v2_13b": ModelType.IFT,
-    "gpt2-xl": ModelType.PT,
-    "lxe/Cerebras-GPT-2.7B-Alpaca-SP": ModelType.FT,
-    "Monero/Manticore-13b-Chat-Pyg-Guanaco": ModelType.FT,
-    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b": ModelType.IFT,
-    "Monero/WizardLM-13b-OpenAssistant-Uncensored": ModelType.IFT,
-    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b": ModelType.IFT,
-    "jzjiao/opt-1.3b-rlhf": ModelType.FT,
-    "HuggingFaceH4/starchat-beta": ModelType.IFT,
-    "KnutJaegersberg/gpt-2-xl-EvolInstruct": ModelType.IFT,
-    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct": ModelType.IFT,
-    "KnutJaegersberg/galactica-orca-wizardlm-1.3b": ModelType.IFT,
-    "openchat/openchat_8192": ModelType.IFT,
-    "openchat/openchat_v2": ModelType.IFT,
-    "openchat/openchat_v2_w": ModelType.IFT,
-    "ausboss/llama-13b-supercot": ModelType.IFT,
-    "ausboss/llama-30b-supercot": ModelType.IFT,
-    "Neko-Institute-of-Science/metharme-7b": ModelType.IFT,
-    "Neko-Institute-of-Science/pygmalion-7b": ModelType.FT,
-    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.IFT,
-    "victor123/WizardLM-13B-1.0": ModelType.IFT,
-    "OpenBuddy/openbuddy-openllama-13b-v7-fp16": ModelType.FT,
-    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16": ModelType.FT,
-    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16": ModelType.FT,
-    "baichuan-inc/Baichuan-7B": ModelType.PT,
-    "tiiuae/falcon-40b-instruct": ModelType.IFT,
-    "tiiuae/falcon-40b": ModelType.PT,
-    "tiiuae/falcon-7b": ModelType.PT,
-    "YeungNLP/firefly-llama-13b": ModelType.FT,
-    "YeungNLP/firefly-llama-13b-v1.2": ModelType.FT,
-    "YeungNLP/firefly-llama2-13b": ModelType.FT,
-    "YeungNLP/firefly-ziya-13b": ModelType.FT,
-    "shaohang/Sparse0.5_OPT-1.3": ModelType.FT,
-    "xzuyn/Alpacino-SuperCOT-13B": ModelType.IFT,
-    "xzuyn/MedicWizard-7B": ModelType.FT,
-    "xDAN-AI/xDAN_13b_l2_lora": ModelType.FT,
-    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.FT,
-    "beomi/llama-2-ko-7b": ModelType.IFT,
-    "Salesforce/codegen-6B-multi": ModelType.PT,
-    "Salesforce/codegen-16B-nl": ModelType.PT,
-    "Salesforce/codegen-6B-nl": ModelType.PT,
-    "ai-forever/rugpt3large_based_on_gpt2": ModelType.FT,
-    "gpt2-large": ModelType.PT,
-    "frank098/orca_mini_3b_juniper": ModelType.FT,
-    "frank098/WizardLM_13B_juniper": ModelType.FT,
-    "FPHam/Free_Sydney_13b_HF": ModelType.FT,
-    "huggingface/llama-13b": ModelType.PT,
-    "huggingface/llama-7b": ModelType.PT,
-    "huggingface/llama-65b": ModelType.PT,
-    "huggingface/llama-30b": ModelType.PT,
-    "Henk717/chronoboros-33B": ModelType.IFT,
-    "jondurbin/airoboros-13b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-7b": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.1": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.2": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.3": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-l2-7b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-l2-13b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-l2-70b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-13b": ModelType.IFT,
-    "jondurbin/airoboros-33b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-33b-gpt4-1.2": ModelType.IFT,
-    "jondurbin/airoboros-65b-gpt4-1.2": ModelType.IFT,
-    "ariellee/SuperPlatty-30B": ModelType.IFT,
-    "danielhanchen/open_llama_3b_600bt_preview": ModelType.FT,
-    "cerebras/Cerebras-GPT-256M": ModelType.PT,
-    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
-    "cerebras/Cerebras-GPT-13B": ModelType.PT,
-    "cerebras/Cerebras-GPT-2.7B": ModelType.PT,
-    "cerebras/Cerebras-GPT-111M": ModelType.PT,
-    "cerebras/Cerebras-GPT-6.7B": ModelType.PT,
-    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf": ModelType.RL,
-    "Yhyu13/llama-30B-hf-openassitant": ModelType.FT,
-    "NousResearch/Nous-Hermes-Llama2-13b": ModelType.IFT,
-    "NousResearch/Nous-Hermes-llama-2-7b": ModelType.IFT,
-    "NousResearch/Redmond-Puffin-13B": ModelType.IFT,
-    "NousResearch/Nous-Hermes-13b": ModelType.IFT,
-    "project-baize/baize-v2-7b": ModelType.IFT,
-    "project-baize/baize-v2-13b": ModelType.IFT,
-    "LLMs/WizardLM-13B-V1.0": ModelType.FT,
-    "LLMs/AlpacaGPT4-7B-elina": ModelType.FT,
-    "wenge-research/yayi-7b": ModelType.FT,
-    "wenge-research/yayi-7b-llama2": ModelType.FT,
-    "wenge-research/yayi-13b-llama2": ModelType.FT,
-    "yhyhy3/open_llama_7b_v2_med_instruct": ModelType.IFT,
-    "llama-anon/instruct-13b": ModelType.IFT,
-    "huggingtweets/jerma985": ModelType.FT,
-    "huggingtweets/gladosystem": ModelType.FT,
-    "huggingtweets/bladeecity-jerma985": ModelType.FT,
-    "huggyllama/llama-13b": ModelType.PT,
-    "huggyllama/llama-65b": ModelType.PT,
-    "FabbriSimo01/Facebook_opt_1.3b_Quantized": ModelType.PT,
-    "upstage/Llama-2-70b-instruct": ModelType.IFT,
-    "upstage/Llama-2-70b-instruct-1024": ModelType.IFT,
-    "upstage/llama-65b-instruct": ModelType.IFT,
-    "upstage/llama-30b-instruct-2048": ModelType.IFT,
-    "upstage/llama-30b-instruct": ModelType.IFT,
-    "WizardLM/WizardLM-13B-1.0": ModelType.IFT,
-    "WizardLM/WizardLM-13B-V1.1": ModelType.IFT,
-    "WizardLM/WizardLM-13B-V1.2": ModelType.IFT,
-    "WizardLM/WizardLM-30B-V1.0": ModelType.IFT,
-    "WizardLM/WizardCoder-15B-V1.0": ModelType.IFT,
-    "gpt2": ModelType.PT,
-    "keyfan/vicuna-chinese-replication-v1.1": ModelType.IFT,
-    "nthngdy/pythia-owt2-70m-100k": ModelType.FT,
-    "nthngdy/pythia-owt2-70m-50k": ModelType.FT,
-    "quantumaikr/KoreanLM-hf": ModelType.FT,
-    "quantumaikr/open_llama_7b_hf": ModelType.FT,
-    "quantumaikr/QuantumLM-70B-hf": ModelType.IFT,
-    "MayaPH/FinOPT-Lincoln": ModelType.FT,
-    "MayaPH/FinOPT-Franklin": ModelType.FT,
-    "MayaPH/GodziLLa-30B": ModelType.IFT,
-    "MayaPH/GodziLLa-30B-plus": ModelType.IFT,
-    "MayaPH/FinOPT-Washington": ModelType.FT,
-    "ogimgio/gpt-neo-125m-neurallinguisticpioneers": ModelType.FT,
-    "layoric/llama-2-13b-code-alpaca": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b-v2": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b-v3": ModelType.FT,
-    "timdettmers/guanaco-33b-merged": ModelType.FT,
-    "elinas/chronos-33b": ModelType.IFT,
-    "heegyu/RedTulu-Uncensored-3B-0719": ModelType.IFT,
-    "heegyu/WizardVicuna-Uncensored-3B-0719": ModelType.IFT,
-    "heegyu/WizardVicuna-3B-0719": ModelType.IFT,
-    "meta-llama/Llama-2-7b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-7b-hf": ModelType.PT,
-    "meta-llama/Llama-2-13b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-13b-hf": ModelType.PT,
-    "meta-llama/Llama-2-70b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-70b-hf": ModelType.PT,
-    "xhyi/PT_GPTNEO350_ATG": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-20b": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt": ModelType.FT,
-    "h2oai/h2ogpt-oig-oasst1-512-6_9b": ModelType.IFT,
-    "h2oai/h2ogpt-oasst1-512-12b": ModelType.IFT,
-    "h2oai/h2ogpt-oig-oasst1-256-6_9b": ModelType.IFT,
-    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt": ModelType.FT,
-    "h2oai/h2ogpt-oasst1-512-20b": ModelType.IFT,
-    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b": ModelType.FT,
-    "bofenghuang/vigogne-13b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-13b-chat": ModelType.FT,
-    "bofenghuang/vigogne-2-7b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-7b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-7b-chat": ModelType.FT,
-    "Vmware/open-llama-7b-v2-open-instruct": ModelType.IFT,
-    "VMware/open-llama-0.7T-7B-open-instruct-v1.1": ModelType.IFT,
-    "ewof/koishi-instruct-3b": ModelType.IFT,
-    "gywy/llama2-13b-chinese-v1": ModelType.FT,
-    "GOAT-AI/GOAT-7B-Community": ModelType.FT,
-    "psyche/kollama2-7b": ModelType.FT,
-    "TheTravellingEngineer/llama2-7b-hf-guanaco": ModelType.FT,
-    "beaugogh/pythia-1.4b-deduped-sharegpt": ModelType.FT,
-    "augtoma/qCammel-70-x": ModelType.IFT,
-    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload": ModelType.IFT,
-    "anhnv125/pygmalion-6b-roleplay": ModelType.FT,
-    "64bits/LexPodLM-13B": ModelType.FT,
-}
-
-
-def model_type_from_str(type):
-    if "fine-tuned" in type or "🔶" in type:
-        return ModelType.FT
-    if "pretrained" in type or "🟢" in type:
-        return ModelType.PT
-    if "RL-tuned" in type or "🟦" in type:
-        return ModelType.RL
-    if "instruction-tuned" in type or "⭕" in type:
-        return ModelType.IFT
-    return ModelType.Unknown

src/display_models/read_results.py

@@ -1,152 +0,0 @@
-import json
-import os
-from dataclasses import dataclass
-from typing import Dict, List, Tuple
-from distutils.util import strtobool
-
-import dateutil
-import numpy as np
-
-from src.display_models.utils import AutoEvalColumn, make_clickable_model
-
-# 현우 - ko_commongen_v2 : acc_norm인지 체크 필요함  
-METRICS = ["acc_norm", "acc_norm", "acc", "mc2", "acc_norm"]
-BENCHMARKS = ["ko_arc_challenge", "ko_hellaswag", "ko_mmlu", "ko_truthfulqa_mc", "ko_commongen_v2"] #, "ethicalverification"]
-BENCH_TO_NAME = {
-    "ko_arc_challenge": AutoEvalColumn.arc.name,
-    "ko_hellaswag": AutoEvalColumn.hellaswag.name,
-    "ko_mmlu": AutoEvalColumn.mmlu.name,
-    "ko_truthfulqa_mc": AutoEvalColumn.truthfulqa.name,
-    "ko_commongen_v2": AutoEvalColumn.commongen_v2.name,
-    # TODO: Uncomment when we have results for these
-    # "ethicalverification": AutoEvalColumn.ethicalverification.name,
-}
-IS_PUBLIC = bool(strtobool(os.environ.get("IS_PUBLIC", "True")))
-
-@dataclass
-class EvalResult:
-    eval_name: str
-    org: str
-    model: str
-    revision: str
-    results: dict
-    precision: str = ""
-    model_type: str = ""
-    weight_type: str = ""
-
-    def to_dict(self):
-        from src.load_from_hub import is_model_on_hub
-
-        if self.org is not None:
-            base_model = f"{self.org}/{self.model}"
-        else:
-            base_model = f"{self.model}"
-        data_dict = {}
-
-        data_dict["eval_name"] = self.eval_name  # not a column, just a save name
-        data_dict["weight_type"] = self.weight_type  # not a column, just a save name
-        data_dict[AutoEvalColumn.precision.name] = self.precision
-        data_dict[AutoEvalColumn.model_type.name] = self.model_type
-        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
-        data_dict[AutoEvalColumn.dummy.name] = base_model
-        data_dict[AutoEvalColumn.revision.name] = self.revision
-        data_dict[AutoEvalColumn.average.name] = sum([v for k, v in self.results.items()]) / 5.0
-        data_dict[AutoEvalColumn.still_on_hub.name] = (
-            is_model_on_hub(base_model, self.revision)[0] or base_model == "baseline"
-        )
-
-        for benchmark in BENCHMARKS:
-            if benchmark not in self.results.keys():
-                self.results[benchmark] = None
-
-        for k, v in BENCH_TO_NAME.items():
-            data_dict[v] = self.results[k]
-
-        return data_dict
-
-
-def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
-    with open(json_filepath) as fp:
-        data = json.load(fp)
-
-    try:
-        config = data["config"]
-    except KeyError:
-        config = data["config_general"]
-    model = config.get("model_name", None)
-    if model is None:
-        model = config.get("model_args", None)
-
-    model_sha = config.get("model_sha", "")
-    model_split = model.split("/", 1)
-
-    precision = config.get("model_dtype")
-
-    model = model_split[-1]
-
-    if len(model_split) == 1:
-        org = None
-        model = model_split[0]
-        result_key = f"{model}_{precision}"
-    else:
-        org = model_split[0]
-        model = model_split[1]
-        result_key = f"{org}_{model}_{precision}"
-
-    eval_results = []
-    for benchmark, metric in zip(BENCHMARKS, METRICS):
-        accs = np.array([v.get(metric, None) for k, v in data["results"].items() if benchmark in k])
-        if accs.size == 0 or any([acc is None for acc in accs]):
-            continue
-        mean_acc = np.mean(accs) * 100.0
-        eval_results.append(
-            EvalResult(
-                eval_name=result_key,
-                org=org,
-                model=model,
-                revision=model_sha,
-                results={benchmark: mean_acc},
-                precision=precision,  # todo model_type=, weight_type=
-            )
-        )
-
-    return result_key, eval_results
-
-
-def get_eval_results(results_path: str) -> List[EvalResult]:
-    json_filepaths = []
-
-    for root, dir, files in os.walk(results_path + ("-private" if not IS_PUBLIC else "")):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        # store results by precision maybe?
-        try:
-            files.sort(key=lambda x: dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        # up_to_date = files[-1]
-        for file in files:
-            json_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for json_filepath in json_filepaths:
-        result_key, results = parse_eval_result(json_filepath)
-        for eval_result in results:
-            if result_key in eval_results.keys():
-                eval_results[result_key].results.update(eval_result.results)
-            else:
-                eval_results[result_key] = eval_result
-
-    eval_results = [v for v in eval_results.values()]
-
-    return eval_results
-
-
-def get_eval_results_dicts(results_path: str) -> List[Dict]:
-    eval_results = get_eval_results(results_path)
-
-    return [e.to_dict() for e in eval_results]

src/display_models/utils.py

@@ -1,149 +0,0 @@
-import os
-from dataclasses import dataclass
-
-from huggingface_hub import HfApi
-
-API = HfApi()
-
-
-# These classes are for user facing column names, to avoid having to change them
-# all around the code when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:  # Auto evals column
-    model_type_symbol = ColumnContent("T", "str", True)
-    model = ColumnContent("Model", "markdown", True)
-    average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("Ko-ARC", "number", True)
-    hellaswag = ColumnContent("Ko-HellaSwag", "number", True)
-    mmlu = ColumnContent("Ko-MMLU", "number", True)
-    truthfulqa = ColumnContent("Ko-TruthfulQA", "number", True)
-    commongen_v2 = ColumnContent("Ko-CommonGen V2", "number", True)
-    # TODO: Uncomment when we have results for these
-    # ethicalverification = ColumnContent("EthicalVerification", "number", True)
-    model_type = ColumnContent("Type", "str", False)
-    precision = ColumnContent("Precision", "str", False)  # , True)
-    license = ColumnContent("Hub License", "str", False)
-    params = ColumnContent("#Params (B)", "number", False)
-    likes = ColumnContent("Hub ❤️", "number", False)
-    still_on_hub = ColumnContent("Available on the hub", "bool", False)
-    revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent(
-        "model_name_for_query", "str", True
-    )  # dummy col to implement search bar (hidden by custom CSS)
-
-
-@dataclass(frozen=True)
-class EloEvalColumn:  # Elo evals column
-    model = ColumnContent("Model", "markdown", True)
-    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
-    human_all = ColumnContent("Human (all)", "number", True)
-    human_instruct = ColumnContent("Human (instruct)", "number", True)
-    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
-
-
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-
-LLAMAS = [
-    "huggingface/llama-7b",
-    "huggingface/llama-13b",
-    "huggingface/llama-30b",
-    "huggingface/llama-65b",
-]
-
-
-KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
-VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
-OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
-DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
-MODEL_PAGE = "https://huggingface.co/models"
-LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
-VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
-ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-
-    if model_name in LLAMAS:
-        link = LLAMA_LINK
-        model_name = model_name.split("/")[1]
-    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
-        link = VICUNA_LINK
-        model_name = "stable-vicuna-13b"
-    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
-        link = ALPACA_LINK
-        model_name = "alpaca-13b"
-    if model_name == "dolly-12b":
-        link = DOLLY_LINK
-    elif model_name == "vicuna-13b":
-        link = VICUNA_LINK
-    elif model_name == "koala-13b":
-        link = KOALA_LINK
-    elif model_name == "oasst-12b":
-        link = OASST_LINK
-
-    details_model_name = model_name.replace("/", "__")
-    # details_link = f"https://huggingface.co/datasets/open-ko-llm-leaderboard/details_{details_model_name}"
-
-    # if not bool(os.getenv("DEBUG", "False")):
-    #     # We only add these checks when not debugging, as they are extremely slow
-    #     print(f"details_link: {details_link}")
-    #     try:
-    #         check_path = list(
-    #             API.list_files_info(
-    #                 repo_id=f"open-ko-llm-leaderboard/details_{details_model_name}",
-    #                 paths="README.md",
-    #                 repo_type="dataset",
-    #             )
-    #         )
-    #         print(f"check_path: {check_path}")
-    #     except Exception as err:
-    #         # No details repo for this model
-    #         print(f"No details repo for this model: {err}")
-    #         return model_hyperlink(link, model_name)
-
-    return model_hyperlink(link, model_name) # + "  " + model_hyperlink(details_link, "📑")
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/envs.py

@@ -0,0 +1,32 @@
+import os
+
+from huggingface_hub import HfApi
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+REPO_ID = "upstage/open-ko-llm-leaderboard"
+QUEUE_REPO = "open-ko-llm-leaderboard/requests"
+RESULTS_REPO = "open-ko-llm-leaderboard/results"
+
+PRIVATE_QUEUE_REPO = "open-ko-llm-leaderboard/private-requests"
+PRIVATE_RESULTS_REPO = "open-ko-llm-leaderboard/private-results"
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+
+EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
+EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
+
+PATH_TO_COLLECTION = "open-ko-llm-leaderboard/ko-llm-leaderboard-best-models-659c7e45a481ceea4c883506"
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+HAS_HIGHER_RATE_LIMIT = []
+
+API = HfApi(token=H4_TOKEN)

src/leaderboard/filter_models.py

@@ -0,0 +1,50 @@
+from src.display.formatting import model_hyperlink
+from src.display.utils import AutoEvalColumn
+
+# Models which have been flagged by users as being problematic for a reason or another
+# (Model name to forum discussion link)
+FLAGGED_MODELS = {
+    "merged": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/510",
+}
+
+# Models which have been requested by orgs to not be submitted on the leaderboard
+DO_NOT_SUBMIT_MODELS = [
+]
+
+
+def flag_models(leaderboard_data: list[dict]):
+    for model_data in leaderboard_data:
+        # Merges are flagged automatically
+        if model_data[AutoEvalColumn.flagged.name] == True:
+            flag_key = "merged"
+        else:
+            flag_key = model_data["model_name_for_query"]
+
+        if flag_key in FLAGGED_MODELS:
+            issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
+            issue_link = model_hyperlink(
+                FLAGGED_MODELS[flag_key],
+                f"See discussion #{issue_num}",
+            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+            model_data[AutoEvalColumn.flagged.name] = True
+        else:
+            model_data[AutoEvalColumn.flagged.name] = False
+
+
+def remove_forbidden_models(leaderboard_data: list[dict]):
+    indices_to_remove = []
+    for ix, model in enumerate(leaderboard_data):
+        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+            indices_to_remove.append(ix)
+
+    for ix in reversed(indices_to_remove):
+        leaderboard_data.pop(ix)
+    return leaderboard_data
+
+
+def filter_models(leaderboard_data: list[dict]):
+    leaderboard_data = remove_forbidden_models(leaderboard_data)
+    flag_models(leaderboard_data)

src/leaderboard/read_evals.py

@@ -0,0 +1,234 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from huggingface_hub import ModelCard
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub, check_model_card
+
+
+@dataclass
+class EvalResult:
+    # Also see src.display.utils.AutoEvalColumn for what will be displayed.
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" # From config file
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+    is_merge: bool = False
+    flagged: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        # We manage the legacy config format
+        config = data.get("config", data.get("config_general", None))
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, error, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # If the model doesn't have a model card or a license, we consider it's deleted
+        if still_on_hub:
+            try:
+                if check_model_card(full_model)[0] is False:
+                    still_on_hub = False
+            except Exception:
+                still_on_hub = False
+
+        # Check if the model is a merge
+        is_merge_from_metadata = False
+        flagged = False
+        if still_on_hub:
+            model_card = ModelCard.load(full_model)
+
+            if model_card.data.tags:
+                is_merge_from_metadata = "merge" in model_card.data.tags
+            merge_keywords = ["mergekit", "merged model", "merge model", "merging", "merge", "merged", "Carbon"]
+            # If the model is a merge but not saying it in the metadata, we flag it
+            is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
+            flagged = is_merge_from_model_card and not is_merge_from_metadata
+
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # Some truthfulQA values are NaNs
+            if task.benchmark == "truthfulqa:mc" and "harness|truthfulqa:mc|0" in data["results"]:
+                if math.isnan(float(data["results"]["harness|truthfulqa:mc|0"][task.metric])):
+                    results[task.benchmark] = 0.0
+                    continue
+
+            # We average all scores of a given metric (mostly for mmlu)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+            is_merge=is_merge_from_metadata,
+            flagged=flagged,
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.merged.name: self.is_merge,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol, # + "🥦" if self.is_merge,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.flagged.name: self.flagged
+
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/{load_from_hub.py → populate.py}

@@ -1,56 +1,30 @@
 import json
 import os
-from collections import defaultdict
 
 import pandas as pd
-from transformers import AutoConfig
 
-from src.assets.hardcoded_evals import baseline
-from src.display_models.get_model_metadata import apply_metadata
-from src.display_models.read_results import get_eval_results_dicts, make_clickable_model
-from src.display_models.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.filter_models import filter_models
+from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_all_requested_models(requested_models_dir: str) -> set[str]:
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+    # all_data_json.append(baseline_row)
+    filter_models(all_data_json)
 
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"): continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates
-
-
-def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    all_data = get_eval_results_dicts(results_path)
-
-    # all_data.append(baseline)
-    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
-
-    df = pd.DataFrame.from_records(all_data)
+    df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+    return raw_data, df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
@@ -85,19 +59,3 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
     df_failed = pd.DataFrame.from_records(failed_list, columns=cols)
     return df_finished[cols], df_running[cols], df_pending[cols], df_failed[cols]
-
-
-
-def is_model_on_hub(model_name: str, revision: str) -> bool:
-    try:
-        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
-        return True, None
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-        )
-
-    except Exception:
-        return False, "was not found on hub!"

src/rate_limiting.py

@@ -1,16 +0,0 @@
-
-from datetime import datetime, timezone, timedelta
-
-
-def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):
-    org_or_user, _ = submission_name.split("/")
-    if org_or_user not in users_to_submission_dates:
-        return 0
-    submission_dates = sorted(users_to_submission_dates[org_or_user])
-
-    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
-    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
-
-    return len(submissions_after_timelimit)
-
-

src/submission/check_validity.py

@@ -0,0 +1,129 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig, AutoTokenizer
+
+from src.envs import HAS_HIGHER_RATE_LIMIT
+
+
+# ht to @Wauplin, thank you for the snippet!
+# See https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/317
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    # Returns operation status, and error message
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) #, force_download=True)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError ):
+        try:
+            size_match = re.search(size_pattern, model_info.modelId.split("/")[-1].lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.split("/")[-1].lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    return model_info.config.get("architectures", "Unknown")
+
+def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
+    if org_or_user not in users_to_submission_dates:
+        return True, ""
+    submission_dates = sorted(users_to_submission_dates[org_or_user])
+
+    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+
+    num_models_submitted_in_period = len(submissions_after_timelimit)
+    if org_or_user in HAS_HIGHER_RATE_LIMIT:
+        rate_limit_quota = 2 * rate_limit_quota
+
+    if num_models_submitted_in_period > rate_limit_quota:
+        error_msg = f"Organisation or user `{org_or_user}`"
+        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
+        error_msg += f"in the last {rate_limit_period} days.\n"
+        error_msg += (
+            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        )
+        return False, error_msg
+    return True, ""
+
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,138 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+    user_submission_permission,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    private: bool,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Upstage models are now allowed to be submitted to ensure the transparency and fairness of the leaderboard.
+    if user_name == "upstage":
+        return styled_warning("We do not conduct evaluations on Upstage models to ensure the transparency and fairness of the leaderboard. Please take this into consideration.")
+
+    # Is the user rate limited?
+    if user_name != "":
+        user_can_submit, error_msg = user_submission_permission(
+            user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+        )
+        if not user_can_submit:
+            return styled_error(error_msg)
+
+    # Did the model authors forbid its submission to the leaderboard?
+    if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
+        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=H4_TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )

src/tools/collections.py

@@ -0,0 +1,82 @@
+import os
+
+import pandas as pd
+from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
+from huggingface_hub.utils._errors import HfHubHTTPError
+from pandas import DataFrame
+
+from src.display.utils import AutoEvalColumn, ModelType
+from src.envs import H4_TOKEN, PATH_TO_COLLECTION
+
+# Specific intervals for the collections
+intervals = {
+    "0~3B": pd.Interval(0, 3, closed="right"),
+    "3~7B": pd.Interval(3, 7.3, closed="right"),
+    "7~13B": pd.Interval(7.3, 13, closed="right"),
+    "13~35B": pd.Interval(13, 35, closed="right"),
+    "35~60B": pd.Interval(35, 60, closed="right"),
+    "60B+": pd.Interval(60, 10000, closed="right"),
+}
+
+def update_collections(df: DataFrame):
+    """This function updates the Open Ko LLM Leaderboard model collection with the latest best models for
+    each size category and type.
+    """
+    collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=H4_TOKEN)
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+
+    cur_best_models = []
+
+    ix = 0
+    for type in ModelType:
+        if type.value.name == "":
+            continue
+        for size in intervals:
+            # We filter the df to gather the relevant models
+            type_emoji = [t[0] for t in type.value.symbol]
+            filtered_df = df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+            numeric_interval = pd.IntervalIndex([intervals[size]])
+            mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+            filtered_df = filtered_df.loc[mask]
+
+            best_models = list(
+                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name]
+            )
+            print(type.value.symbol, size, best_models[:10])
+
+            # We add them one by one to the leaderboard
+            for model in best_models:
+                ix += 1
+                cur_len_collection = len(collection.items)
+                try:
+                    collection = add_collection_item(
+                        PATH_TO_COLLECTION,
+                        item_id=model,
+                        item_type="model",
+                        exists_ok=True,
+                        note=f"Best {type.to_str(' ')} model of size {size} on the leaderboard today!",
+                        token=H4_TOKEN,
+                    )
+                    if (
+                        len(collection.items) > cur_len_collection
+                    ):  # we added an item - we make sure its position is correct
+                        item_object_id = collection.items[-1].item_object_id
+                        update_collection_item(
+                            collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix
+                        )
+                        cur_len_collection = len(collection.items)
+                    cur_best_models.append(model)
+                    break
+                except HfHubHTTPError:
+                    continue
+
+    collection = get_collection(PATH_TO_COLLECTION, token=H4_TOKEN)
+    for item in collection.items:
+        if item.item_id not in cur_best_models:
+            try:
+                delete_collection_item(
+                    collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
+                )
+            except HfHubHTTPError:
+                continue

src/tools/model_backlinks.py

@@ -0,0 +1,3 @@
+models = [
+    "baseline",
+]

src/tools/plots.py

@@ -0,0 +1,154 @@
+import pandas as pd
+import numpy as np
+import plotly.express as px
+from plotly.graph_objs import Figure
+
+from src.leaderboard.filter_models import FLAGGED_MODELS
+from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, Tasks, Task, BENCHMARK_COLS
+from src.leaderboard.read_evals import EvalResult
+
+
+
+def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
+    """
+    Generates a DataFrame containing the maximum scores until each date.
+
+    :param results_df: A DataFrame containing result information including metric scores and dates.
+    :return: A new DataFrame containing the maximum scores until each date for every metric.
+    """
+    # Step 1: Ensure 'date' is in datetime format and sort the DataFrame by it
+    results_df = pd.DataFrame(raw_data)
+    #results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
+    results_df.sort_values(by="date", inplace=True)
+
+    # Step 2: Initialize the scores dictionary
+    scores = {k: [] for k in BENCHMARK_COLS + [AutoEvalColumn.average.name]}
+
+    # Step 3: Iterate over the rows of the DataFrame and update the scores dictionary
+    for task in [t.value for t in Tasks] + [Task("Average", "avg", AutoEvalColumn.average.name)]:
+        current_max = 0
+        last_date = ""
+        column = task.col_name
+        for _, row in results_df.iterrows():
+            current_model = row["full_model"]
+            if current_model in FLAGGED_MODELS:
+                continue
+
+            current_date = row["date"]
+            if task.benchmark == "Average":
+                current_score = np.mean(list(row["results"].values()))
+            else:
+                current_score = row["results"][task.benchmark]
+
+            if current_score > current_max:
+                if current_date == last_date and len(scores[column]) > 0:
+                    scores[column][-1] = {"model": current_model, "date": current_date, "score": current_score}
+                else:
+                    scores[column].append({"model": current_model, "date": current_date, "score": current_score})
+                current_max = current_score
+                last_date = current_date
+
+    # Step 4: Return all dictionaries as DataFrames
+    return {k: pd.DataFrame(v) for k, v in scores.items()}
+
+
+def create_plot_df(scores_df: dict[str: pd.DataFrame]) -> pd.DataFrame:
+    """
+    Transforms the scores DataFrame into a new format suitable for plotting.
+
+    :param scores_df: A DataFrame containing metric scores and dates.
+    :return: A new DataFrame reshaped for plotting purposes.
+    """
+    # Initialize the list to store DataFrames
+    dfs = []
+
+    # Iterate over the cols and create a new DataFrame for each column
+    for col in BENCHMARK_COLS + [AutoEvalColumn.average.name]:
+        d = scores_df[col].reset_index(drop=True)
+        d["task"] = col
+        dfs.append(d)
+
+    # Concatenate all the created DataFrames
+    concat_df = pd.concat(dfs, ignore_index=True)
+
+    # Sort values by 'date'
+    concat_df.sort_values(by="date", inplace=True)
+    concat_df.reset_index(drop=True, inplace=True)
+    return concat_df
+
+
+def create_metric_plot_obj(
+    df: pd.DataFrame, metrics: list[str], title: str
+) -> Figure:
+    """
+    Create a Plotly figure object with lines representing different metrics
+    and horizontal dotted lines representing human baselines.
+
+    :param df: The DataFrame containing the metric values, names, and dates.
+    :param metrics: A list of strings representing the names of the metrics
+                    to be included in the plot.
+    :param title: A string representing the title of the plot.
+    :return: A Plotly figure object with lines representing metrics and
+             horizontal dotted lines representing human baselines.
+    """
+
+    # Filter the DataFrame based on the specified metrics
+    df = df[df["task"].isin(metrics)]
+
+    # Filter the human baselines based on the specified metrics
+    filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
+
+    # Create a line figure using plotly express with specified markers and custom data
+    fig = px.line(
+        df,
+        x="date",
+        y="score",
+        color="task",
+        markers=True,
+        custom_data=["task", "score", "model"],
+        title=title,
+    )
+
+    # Update hovertemplate for better hover interaction experience
+    fig.update_traces(
+        hovertemplate="<br>".join(
+            [
+                "Model Name: %{customdata[2]}",
+                "Metric Name: %{customdata[0]}",
+                "Date: %{x}",
+                "Metric Value: %{y}",
+            ]
+        )
+    )
+
+    # Update the range of the y-axis
+    fig.update_layout(yaxis_range=[0, 100])
+
+    # Create a dictionary to hold the color mapping for each metric
+    metric_color_mapping = {}
+
+    # Map each metric name to its color in the figure
+    for trace in fig.data:
+        metric_color_mapping[trace.name] = trace.line.color
+
+    # Iterate over filtered human baselines and add horizontal lines to the figure
+    for metric, value in filtered_human_baselines.items():
+        color = metric_color_mapping.get(metric, "blue")  # Retrieve color from mapping; default to blue if not found
+        location = "top left" if metric == "Ko-HellaSwag" else "bottom left"  # Set annotation position
+        # Add horizontal line with matched color and positioned annotation
+        fig.add_hline(
+            y=value,
+            line_dash="dot",
+            annotation_text=f"{metric} human baseline",
+            annotation_position=location,
+            annotation_font_size=10,
+            annotation_font_color=color,
+            line_color=color,
+        )
+
+    return fig
+
+
+# Example Usage:
+# human_baselines dictionary is defined.
+# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")