added model submission functionality

app.py

@@ -27,13 +27,14 @@ from src.display.utils import (
     AutoEvalColumn,
     ModelType,
     ModelArch,
+    PromptTemplateName,
     Precision,
     WeightType,
     fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.submission.submit import add_new_eval, PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG
 
 
 def restart_space():
@@ -155,7 +156,61 @@ def filter_models(
 
     return filtered_df
 
+def change_submit_request_form(model_architecture):
+    match model_architecture:
+        case "Encoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=False), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=False
+                    )
+            )
+        case "Decoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=False), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=True
+                    )
+            )
+        case "GLiNER Encoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=True), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=True
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=False
+                    )
+            )
 
+            
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -370,6 +425,108 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
+                with gr.Column():
+                    with gr.Accordion(
+                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+                    with gr.Accordion(
+                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+
+                    with gr.Accordion(
+                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+
+                    model_name_textbox = gr.Textbox(label="Model name")
+
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+
+                    model_arch = gr.Radio(
+                        choices=[t.to_str(" : ") for t in ModelArch if t != ModelArch.Unknown],
+                        label="Model Architecture",
+                    )
+                    
+                    model_type = gr.Dropdown(
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        label="Model type",
+                        multiselect=False,
+                        value=None,
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    label_normalization_map = gr.Textbox(lines=6, label="Label Normalization Map", placeholder=PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG)
+                    gliner_threshold = gr.Textbox(label="Threshold for GLiNER models", visible=False)
+                    gliner_tokenizer_bool = gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
+                    )
+                    prompt_name = gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=False
+                    )# should be a dropdown
+
+                    # parsing_function - this is tied to the prompt & therefore does not need to be specified
+                    # generation_parameters = gr.Textbox(label="Generation params in json format") just default for now
+
+                    model_arch.change(fn=change_submit_request_form, inputs=model_arch, outputs=[
+                        gliner_threshold, 
+                        gliner_tokenizer_bool, 
+                        prompt_name])
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    # base_model_name_textbox,
+                    revision_name_textbox,
+                    model_arch,
+                    label_normalization_map,
+                    gliner_threshold,
+                    gliner_tokenizer_bool,
+                    prompt_name,
+                    # weight_type,
+                    model_type,
+                ],
+                submission_result,
+            )
+
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

src/about.py

@@ -43,27 +43,32 @@ NUM_FEWSHOT = 0  # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title"> NER Leaderboard</h1>"""
+TITLE = """""" #<h1 align="center" id="space-title"> NER Leaderboard</h1>"""
 LOGO = """<img src="file/assets/image.png" alt="Clinical X HF" width="500" height="333">"""
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Named Entity Recognition of clinical entities is crucial for advancing natural language processing (NLP) applications in healthcare as it is foundational for tasks such as information extraction, clinical decision support, and automated documentation. 
-The datasets used for this evaluation encompass a wide range of medical entities, including diseases, symptoms, medications, procedures and anatomical terms. These datasets are sourced from openly available clinical data (including annotations) to ensure comprehensive coverage and reflect the complexity of real-world medical language. More details about the datasets included can be found in the "About" section.
-The evaluation metrics used in this leaderboard focus primarily on the F1-score, a widely recognized measure of a model's accuracy. More details about the evaluation metric can be found in the "About" section
+The main goal of the Named Clinical Entity Recognition Leaderboard is to evaluate and benchmark the performance of various language models in accurately identifying and classifying named clinical entities across diverse medical domains. This task is crucial for advancing natural language processing (NLP) applications in healthcare, as accurate entity recognition is foundational for tasks such as information extraction, clinical decision support, and automated documentation.  
+
+The datasets used for this evaluation encompass a wide range of medical entities, including diseases, symptoms, medications, procedures and anatomical terms. These datasets are sourced from openly available clinical data (including annotations) to ensure comprehensive coverage and reflect the complexity of real-world medical language. More details about the datasets included can be found below ("About" section).  
+
+The evaluation metrics used in this leaderboard focus primarily on the F1-score, a widely recognized measure of a model's accuracy. The different modes of evaluation are also described below.  
+
+Disclaimer: It is important to note that the purpose of this evaluation is purely academic and exploratory. The models assessed here have not been approved for clinical use, and their results should not be interpreted as clinically validated. The leaderboard serves as a platform for researchers to compare models, understand their strengths and limitations, and drive further advancements in the field of clinical NLP.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
 
-#### Disclaimer & Advisory
-
-It is important to note that the purpose of this evaluation is purely academic and exploratory. The models assessed here have not been approved for clinical use, and their results should not be interpreted as clinically validated. The leaderboard serves as a platform for researchers to compare models, understand their strengths and limitations, and drive further advancements in the field of clinical NLP.
-
-## About
 The Named Clinical Entity Recognition Leaderboard is aimed at advancing the field of natural language processing in healthcare. It provides a standardized platform for evaluating and comparing the performance of various language models in recognizing named clinical entities, a critical task for applications such as clinical documentation, decision support, and information extraction. By fostering transparency and facilitating benchmarking, the leaderboard's goal is to drive innovation and improvement in NLP models. It also helps researchers identify the strengths and weaknesses of different approaches, ultimately contributing to the development of more accurate and reliable tools for clinical use. Despite its exploratory nature, the leaderboard aims to play a role in guiding research and ensuring that advancements are grounded in rigorous and comprehensive evaluations. 
+## About
 
 ## How it works
 
+### Evaluation method and metrics
+When training a Named Entity Recognition (NER) system, the most common evaluation methods involve measuring precision, recall, and F1-score at the token level. While these metrics are useful for fine-tuning the NER system, evaluating the predicted named entities for downstream tasks requires metrics at the full named-entity level. We include both evaluation methods: token-based and span-based. We provide an example below which helps in understanding the difference between the methods.
+Example Sentence: "The patient was diagnosed with a skin cancer disease."
+For simplicity, let's assume the an example sentence which contains 10 tokens, with a single two-token disease entity (as shown in the figure below).
+
 ### Datasets
 📈 We evaluate the models on 4 datasets, encompassing 6 entity types
 - [NCBI](https://huggingface.co/datasets/m42-health/clinical_ncbi) 
@@ -81,7 +86,30 @@ To reproduce our results, follow the steps detailed [here](https://github.com/Wa
 """
 
 EVALUATION_QUEUE_TEXT = """
-Follow the steps detailed in the [medics_ner](https://github.com/WadoodAbdul/medics_ner/blob/master/docs/submit_to_leaderboard.md) repo to upload you model to the leaderoard.
+
+Currently, the benchmark supports evaluation for models hosted on the huggingface hub and of type encoder, decoder or gliner type models.
+If your model needs a custom implementation, follow the steps outlined in the [medics_ner](https://github.com/WadoodAbdul/medics_ner/blob/master/docs/custom_model_implementation.md) repo or reach out to our team!
+
+
+### Fields Explanation
+
+#### Model Type:
+- Fine-Tuned: If the training data consisted of any split/variation of the datasets on the leaderboard.  
+- Zero-Shot: If the model did not have any exposure to the datasets on the leaderboard while training.
+
+#### Model Architecture:
+- Encoder: The standard transformer encoder architecture with a token classification head on top.  
+- Decoder: Transformer based autoregressive token generation model.  
+- GLiNER: Architecture outlined in the [GLiNER Paper](https://arxiv.org/abs/2311.08526)
+
+#### Label Normalization Map:
+Not all models have been tuned to output the ner label names in the clinical datasets on this leaderboard. Some models cater to the same entity names with a synonym of it.
+The normalization map can be used to ensure that the models's output are aligned with the labels expected in the datasets.
+
+Note: Multiple model labels can be mapped to a single entity type in the leaderboard dataset. Ex: 'synonym' and 'disease' to 'condition'
+
+
+Upon successful submission of your request, your model's result would be updated on the leaderboard within 5 working days!
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"

src/display/utils.py

@@ -60,8 +60,9 @@ class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    architecture = ColumnContent("model_architecture", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
 
@@ -104,7 +105,7 @@ class ModelType(Enum):
 class ModelArch(Enum):
     Encoder = ModelDetails("Encoder")
     Decoder = ModelDetails("Decoder")
-    EncoderDecoder = ModelDetails("EncoderDecoder")
+    GLiNEREncoder = ModelDetails("GLiNER Encoder")
     Unknown = ModelDetails(name="Other", symbol="?")
 
     def to_str(self, separator=" "):
@@ -116,8 +117,8 @@ class ModelArch(Enum):
             return ModelArch.Encoder
         if "decoder" in type:
             return ModelArch.Decoder
-        if "encoder-decoder" in type:
-            return ModelArch.EncoderDecoder
+        if "GLiNEREncoder" in type:
+            return ModelArch.GLiNEREncoder
         # if "unknown" in type:
         #     return ModelArch.Unknown
         return ModelArch.Unknown
@@ -154,6 +155,14 @@ class Precision(Enum):
         return Precision.Unknown
 
 
+class PromptTemplateName(Enum):
+    UniversalNERTemplate = "universal_ner"
+    LLMHTMLHighlightedSpansTemplate = "llm_html_highlighted_spans"
+    LLamaNERTemplate = "llama_70B_ner_v0.3"
+    MixtralNERTemplate = "mixtral_ner_v0.3.jinja"
+
+
+
 # Column selection
 DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.clinical_type_col]
 Clinical_TYPES_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col]

src/submission/check_validity.py

@@ -59,14 +59,24 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         return False, "was not found on hub!", None
 
 
-def get_model_size(model_info: ModelInfo, precision: str):
+def get_model_size(model_info: ModelInfo, precision: str=None):
     """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
+
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+        try:
+            size_match = re.search(size_pattern, model_info.id.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    if precision:
+        size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    else:
+        size_factor = 1
     model_size = size_factor * model_size
     return model_size
 
@@ -88,7 +98,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(f"{info['model']}_{info['revision']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -1,5 +1,6 @@
 import json
 import os
+import ast
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
@@ -10,18 +11,57 @@ from src.submission.check_validity import (
     get_model_size,
     is_model_on_hub,
 )
+from src.display.utils import PromptTemplateName
 
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
+PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG = """{
+    "NCBI" : {
+        "" : "condition"
+    },
+    "CHIA" : {
+        "" : "condition"
+        "" : "drug"
+        "" : "procedure"
+        "" : "measurement"
+    },
+    "BIORED" : {
+        "" : "condition"
+        "" : "drug"
+        "" : "gene"
+        "" : "gene variant"
+    },
+    "BC5CDR" : {
+        "" : "condition"
+        "" : "drug"
+    }
+}
+
+"""
+
 def add_new_eval(
     model: str,
-    base_model: str,
+    # base_model: str,
     revision: str,
-    precision: str,
-    weight_type: str,
+    # precision: str,
+    # weight_type: str,
+    model_arch: str,
+    label_normalization_map: str,
+    gliner_threshold:str,
+    gliner_tokenizer_bool:str,
+    prompt_template_name:str,
     model_type: str,
 ):
+    """
+    Saves request if valid else returns the error.
+    Validity is checked based on -
+        - model's existence on hub
+        - necessary info on the model's card
+        - label normalization is a valid python dict and contains the keys for all datasets
+        - threshold for gliner is a valid float
+
+    """
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
@@ -33,26 +73,32 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    precision = precision.split(" ")[0]
+    # precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
+    
+    model_type = model_type.split(":")[-1].strip()
 
     # Does the model actually exist?
     if revision == "":
         revision = "main"
 
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    # # Is the model on the hub?
+    # if weight_type in ["Delta", "Adapter"]:
+    #     base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+    #     if not base_model_on_hub:
+    #         return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
+    if not model_arch == "GLiNER Encoder":
         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
+    else:
+        if len(list(API.list_models(model_name=model))) !=1:
+            return styled_error(f'Model "{model}" does not exist on the hub!')        
+    
 
     # Is the model info correctly filled?
     try:
@@ -60,7 +106,7 @@ def add_new_eval(
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = get_model_size(model_info=model_info)
 
     # Were the model card and license filled?
     try:
@@ -72,15 +118,52 @@ def add_new_eval(
     if not modelcard_OK:
         return styled_error(error_msg)
 
+    # Verify the inference config now
+    try:
+        label_normalization_map = ast.literal_eval(label_normalization_map)
+    except Exception as e:
+        return styled_error("Please enter a valid json for the labe; normalization map")
+
+    inference_config = {
+        # "model_arch" : model_arch,
+        "label_normalization_map": label_normalization_map,
+    }
+
+    match model_arch:
+        case "Encoder":
+            pass
+        case "Decoder":
+            if not prompt_template_name in [prompt_template.value for prompt_template in PromptTemplateName]:
+                return styled_error("Prompt template name is invalid")
+            inference_config = {
+                **inference_config,
+                "prompt_template_name": prompt_template_name,
+            }
+        case "GLiNER Encoder":
+                try:
+                    gliner_threshold = float(gliner_threshold)
+                    gliner_tokenizer_bool = ast.literal_eval(gliner_tokenizer_bool)
+                    inference_config = {
+                        **inference_config,
+                        "gliner_threshold": gliner_threshold,
+                        "gliner_tokenizer_bool" : gliner_tokenizer_bool
+                    }
+                except Exception as e:
+                    return styled_error("Please enter a valid float for the threshold")
+        case _:
+            return styled_error("Model Architecture is invalid")
+
     # Seems good, creating the eval
     print("Adding new eval")
 
+
     eval_entry = {
         "model": model,
-        "base_model": base_model,
+        # "base_model": base_model,
         "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+        # "precision": precision,
+        # "weight_type": weight_type,
+        "model_architecture": model_arch,
         "status": "PENDING",
         "submitted_time": current_time,
         "model_type": model_type,
@@ -88,16 +171,18 @@ def add_new_eval(
         "params": model_size,
         "license": license,
         "private": False,
+        "inference_config":inference_config,
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
+
+    if f"{model}_{revision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted. Add the revision if the model has been updated.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_{revision}_eval_request.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))