fix-memory-requirements-for-cpu

app.py

@@ -67,13 +67,10 @@ with demo:
                         search_bar, columns_checkboxes, leaderboard_table = (
                             create_leaderboard_table(open_llm_perf_df)
                         )
-                    if (
-                        config.hardware_provider != "intel"
-                    ):  # TODO intel CPU does not measure the memory requirements correctly, so disable the graph feature until we fix the underlying issue
-                        with gr.TabItem("Find Your Best Model 🧭", id=1):
-                            lat_score_mem_plot = create_lat_score_mem_plot(
-                                open_llm_perf_df
-                            )
+                    with gr.TabItem("Find Your Best Model 🧭", id=1):
+                        lat_score_mem_plot = create_lat_score_mem_plot(
+                            open_llm_perf_df
+                        )
                     ###################### ATTENTIONS SPEEDUP TAB #######################
                     # with gr.TabItem("Attention 📈", id=2):
                     #     attn_prefill_plot, attn_decode_plot = create_attn_plots(

hardware.yaml

@@ -39,7 +39,7 @@
 - machine: 32vCPU-C7i
   description: Intel-Xeon-SPR-385W 🖥️
   detail: |
-    We tested the [32vCPU AWS C7i](https://aws.amazon.com/ec2/instance-types/c7i/) instance for the benchmark.
+    We tested the [32vCPU AWS C7i](https://aws.amazon.com/ec2/instance-types/c7i/) instance for the benchmark. The memory requirement is the max RAM consumption during the decode phase.
   hardware_provider: intel
   hardware_type: cpu
   subsets:

src/llm_perf.py

@@ -15,7 +15,6 @@ COLUMNS_MAPPING = {
     "report.per_token.latency.p50": "Per Token (s)",
     "report.decode.throughput.value": "Decode (tokens/s)",
     "report.decode.efficiency.value": "Energy (tokens/kWh)",
-    "report.decode.memory.max_allocated": "Memory (MB)",
     # deployment settings
     "config.backend.name": "Backend 🏭",
     "config.backend.torch_dtype": "Precision 📥",
@@ -28,6 +27,15 @@ COLUMNS_MAPPING = {
     "Average ⬆️": "Open LLM Score (%)",
     "#Params (B)": "Params (B)",
 }
+
+CUDA_COLUMNS_MAPPING = COLUMNS_MAPPING | {
+    "report.decode.memory.max_allocated": "Memory (MB)",
+}
+
+INTEL_COLUMNS_MAPPING = COLUMNS_MAPPING | {
+    "report.decode.memory.max_ram": "Memory (MB)",
+}
+
 SORTING_COLUMNS = ["Open LLM Score (%)", "Decode (tokens/s)", "Prefill (s)"]
 SORTING_ASCENDING = [False, True, False]
 
@@ -39,9 +47,10 @@ def get_raw_llm_perf_df(
     for subset in subsets:
         for backend in backends:
             try:
+                url = f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/perf-df-{backend}-{hardware_type}-{subset}-{machine}.csv"
                 dfs.append(
                     pd.read_csv(
-                        f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/perf-df-{backend}-{hardware_type}-{subset}-{machine}.csv"
+                        url
                     )
                 )
             except Exception:
@@ -70,7 +79,7 @@ def get_raw_llm_perf_df(
     return llm_perf_df
 
 
-def processed_llm_perf_df(llm_perf_df):
+def processed_llm_perf_df(llm_perf_df, hardware_type: str):
     # some assertions
     assert llm_perf_df["config.scenario.input_shapes.batch_size"].nunique() == 1
     assert llm_perf_df["config.scenario.input_shapes.sequence_length"].nunique() == 1
@@ -105,15 +114,23 @@ def processed_llm_perf_df(llm_perf_df):
             "report.decode.throughput.value": 3,
             "report.decode.efficiency.value": 3,
             "report.decode.memory.max_allocated": 3,
+            "report.decode.memory.max_ram": 3,
             "Average ⬆️": 3,
             "prefill+decode": 3,
             "#Params (B)": 3,
         }
     )
+
     # filter columns
-    llm_perf_df = llm_perf_df[list(COLUMNS_MAPPING.keys())]
-    # rename columns
-    llm_perf_df.rename(columns=COLUMNS_MAPPING, inplace=True)
+    if hardware_type == "cuda":
+        llm_perf_df = llm_perf_df[list(CUDA_COLUMNS_MAPPING.keys())]
+        llm_perf_df.rename(columns=CUDA_COLUMNS_MAPPING, inplace=True)
+    elif hardware_type == "cpu":
+        llm_perf_df = llm_perf_df[list(INTEL_COLUMNS_MAPPING.keys())]
+        llm_perf_df.rename(columns=INTEL_COLUMNS_MAPPING, inplace=True)
+    else:
+        raise ValueError(f"Hardware type {hardware_type} not supported")
+
     # sort by metric
     llm_perf_df.sort_values(
         by=SORTING_COLUMNS,
@@ -121,6 +138,9 @@ def processed_llm_perf_df(llm_perf_df):
         inplace=True,
     )
 
+    assert llm_perf_df["Memory (MB)"].notna().any(), "The dataset should contain at least one memory value, otherwise this implies that all the benchmarks have failed (contains only a traceback)"
+    assert llm_perf_df.columns.is_unique, "All columns should be unique"
+
     return llm_perf_df
 
 
@@ -137,7 +157,7 @@ def get_llm_perf_df(
     else:
         print(f"Dataset machine {machine} not found, downloading...")
         llm_perf_df = get_raw_llm_perf_df(machine, subsets, backends, hardware_type)
-        llm_perf_df = processed_llm_perf_df(llm_perf_df)
+        llm_perf_df = processed_llm_perf_df(llm_perf_df, hardware_type)
         llm_perf_df.to_csv(
             f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False
         )