update

tasks.py

@@ -59,7 +59,7 @@ class Task:
     dataset_name: str | tuple[str, str] = ("gsm8k", "main")
     split: str = "test"
     # metrics: list[str] = field(default_factory=list)
-    metric_name: str | tuple[str, str] = ("sustech/tlem", "gsm8k")
+    metric_name: str | tuple[str, str] = ("sustech/tlem", "mmlu")
     input_column: str = "question"
     label_column: str = ""
     prompt: Optional[Callable | str] = None
@@ -147,12 +147,12 @@ class Task:
             if isinstance(self.metric_name, str)
             else load(*self.metric_name)
         )
-        return metric
+        return metric._compute
 
     @cached_property
     def result(self) -> dict:
         assert self.outputs, "Please run the task first."
-        results = self.metric._compute(
+        results = self.metric(
             responses=self.outputs, references=self.dataset[self.label_column]
         )
         # logging.info(f"{self.name}:{results}")

tlem.py

@@ -72,6 +72,17 @@ class ReasoningMetric(evaluate.Metric):
 class Suite(EvaluationSuite):
     task_class = Task
     utils = utils
+    supported_datasets = [
+        "arc",
+        "hellaswag",
+        "mmlu-chat",
+        "winogrande",
+        "gsm8k",
+        "cmmlu-chat",
+        "ceval-chat",
+        "bbh",
+        "drop",
+    ]
 
     def __getitem__(self, key) -> Task:
         match key:
@@ -171,19 +182,26 @@ class Suite(EvaluationSuite):
                 ]:
                     suite[name] = self.get_suite(name)
             case "tlem":
-                suite = {}
-                for name in [
-                    "arc",
-                    "hellaswag",
-                    "mmlu-chat",
-                    "winogrande",
-                    "gsm8k",
-                    "cmmlu-chat",
-                    "ceval-chat",
-                    # "truthful_qa",
-                    "drop",
-                ]:
-                    suite[name] = self.get_suite(name)
+                suite = {
+                    name: self.get_suite(name)
+                    for name in [
+                        "arc",
+                        "hellaswag",
+                        "mmlu-chat",
+                        "winogrande",
+                        "gsm8k",
+                        "cmmlu-chat",
+                        "ceval-chat",
+                        "bbh",
+                    ]
+                }
+            case "all":
+                suite = {name: self.get_suite(name) for name in self.supported_datasets}
+            case _:
+                raise NotImplementedError(
+                    f"{name} is not supported in {self.supported_datasets}"
+                )
+
         if isinstance(suite, Task):
             suite = [suite]
         if isinstance(suite, list):
@@ -195,6 +213,7 @@ class Suite(EvaluationSuite):
         try:
             return self.tasks[self.tasks.index(task)]
         except ValueError:
+            logging.debug(f"add {task.name} to suite.")
             self.tasks.append(task)
             return self.tasks[-1]
 
@@ -212,6 +231,7 @@ class Suite(EvaluationSuite):
     def load(self, name):
         self.suite.update(self.get_suite(name))
         self.suite = self.drop_duplicates(self.suite)
+        # return self
 
     def __init__(self, name="tlem"):
         super().__init__(name)