src/display/about.py

from dataclasses import dataclass
from enum import Enum

@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str


# Init: to update with your specific keys
class Tasks(Enum):
    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
    # task0 = Task("task_name1", "metric_name", "First task")
    # task1 = Task("task_name2", "metric_name", "Second task")
    Analytical_Talent_LSS = Task("Analytical Talent LSS", "Acc", "Analytical Talent LSS")
    Calculus_USS = Task("Calculus USS", "Acc", "Calculus USS")
    Chemistry_USS = Task("Chemistry USS", "Acc", "Chemistry USS")
    Discrete_Mathematics_USS = Task("Discrete Mathematics USS", "Acc", "Discrete Mathematics USS")
    Economy_USS = Task("Economy USS", "Acc", "Economy USS")
    Geography_USS = Task("Geography USS", "Acc", "Geography USS")
    Geology_USS = Task("Geology USS", "Acc", "Geology USS")
    Geometry_USS = Task("Geometry USS", "Acc", "Geometry USS")
    History_USS = Task("History USS", "Acc", "History USS")
    Logic_USS = Task("Logic USS", "Acc", "Logic USS")
    Mathematical_and_Logical_Intelligence_UPS = Task("Mathematical and Logical Intelligence UPS", "Acc", "Mathematical and Logical Intelligence UPS")
    Mathematics_LPS = Task("Mathematics LPS", "Acc", "Mathematics LPS")
    Mathematics_LSS = Task("Mathematics LSS", "Acc", "Mathematics LSS")
    Mathematics_UPS = Task("Mathematics UPS", "Acc", "Mathematics UPS")
    Mathematics_USS = Task("Mathematics USS", "Acc", "Mathematics USS")
    Mathematics_and_Statistics_USS = Task("Mathematics and Statistics USS", "Acc", "Mathematics and Statistics USS")
    Natural_Sciences_LPS = Task("Natural Sciences LPS", "Acc", "Natural Sciences LPS")
    Natural_Sciences_LSS = Task("Natural Sciences LSS", "Acc", "Natural Sciences LSS")
    Natural_Sciences_UPS = Task("Natural Sciences UPS", "Acc", "Natural Sciences UPS")
    Persian_Literature_LPS = Task("Persian Literature LPS", "Acc", "Persian Literature LPS")
    Persian_Literature_LSS = Task("Persian Literature LSS", "Acc", "Persian Literature LSS")
    Persian_Literature_UPS = Task("Persian Literature UPS", "Acc", "Persian Literature UPS")
    Persian_Literature_USS = Task("Persian Literature USS", "Acc", "Persian Literature USS")
    Philosophy_USS = Task("Philosophy USS", "Acc", "Philosophy USS")
    Physics_USS = Task("Physics USS", "Acc", "Physics USS")
    Probability_and_Statistics_USS = Task("Probability and Statistics USS", "Acc", "Probability and Statistics USS")
    Psychology_USS = Task("Psychology USS", "Acc", "Psychology USS")
    Social_Studies_LPS = Task("Social Studies LPS", "Acc", "Social Studies LPS")
    Social_Studies_LSS = Task("Social Studies LSS", "Acc", "Social Studies LSS")
    Social_Studies_UPS = Task("Social Studies UPS", "Acc", "Social Studies UPS")
    Sociology_USS = Task("Sociology USS", "Acc", "Sociology USS")
    Speed_and_Accuracy_UPS = Task("Speed and Accuracy UPS", "Acc", "Speed and Accuracy UPS")
    Theology_LPS = Task("Theology LPS", "Acc", "Theology LPS")
    Theology_LSS = Task("Theology LSS", "Acc", "Theology LSS")
    Theology_UPS = Task("Theology UPS", "Acc", "Theology UPS")
    Theology_USS = Task("Theology USS", "Acc", "Theology USS")
    Verbal_and_Linguistic_Intelligence_UPS = Task("Verbal and Linguistic Intelligence UPS", "Acc", "Verbal and Linguistic Intelligence UPS")
    Biology_USS = Task("‌Biology USS", "Acc", "‌Biology USS")
    # Avg_on_all_tasks = Task("Avg on all tasks", "Acc", "Avg on all tasks")
    # Avg_on_all_questions = Task("Avg on all questions", "Acc", "Avg on all questions")


# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">Khayyam Challenge (PersianMMLU)</h1>"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = f""""""
# Intro text
# """

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = f""""""
# ## How it works

# ## Reproducibility
# To reproduce our results, here is the commands you can run:

# """

EVALUATION_QUEUE_TEXT = """In progress"""
# ## Some good practices before submitting a model

# ### 1) Make sure you can load your model and tokenizer using AutoClasses:
# ```python
# from transformers import AutoConfig, AutoModel, AutoTokenizer
# config = AutoConfig.from_pretrained("your model name", revision=revision)
# model = AutoModel.from_pretrained("your model name", revision=revision)
# tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
# ```
# If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.

# Note: make sure your model is public!
# Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!

# ### 2) Convert your model weights to [safetensors](https://huggingface.co./docs/safetensors/index)
# It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

# ### 3) Make sure your model has an open license!
# This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗

# ### 4) Fill up your model card
# When we add extra information about models to the leaderboard, it will be automatically taken from the model card

# ## In case of model failure
# If your model is displayed in the `FAILED` category, its execution stopped.
# Make sure you have followed the above steps first.
# If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
# """

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@article{ghahroodi2024khayyam,
  title={Khayyam Challenge (PersianMMLU): Is Your LLM Truly Wise to The Persian Language?},
  author={Ghahroodi, Omid and Nouri, Marzia and Sanian, Mohammad Vali and Sahebi, Alireza and Dastgheib, Doratossadat and Asgari, Ehsaneddin and Baghshah, Mahdieh Soleymani and Rohban, Mohammad Hossein},
  journal={arXiv preprint arXiv:2404.06644},
  year={2024}
}
"""