File size: 5,502 Bytes
25557b5 05c90f4 5b0b818 05c90f4 25557b5 c660995 25557b5 05c90f4 c660995 25557b5 05c90f4 25557b5 e29ab28 05c90f4 e29ab28 05c90f4 e29ab28 25557b5 05c90f4 e29ab28 05c90f4 25557b5 e611814 05c90f4 25557b5 e611814 05c90f4 e611814 05c90f4 e611814 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
import json
import gradio as gr
import pandas as pd
from huggingface_hub import HfFileSystem
RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results"
EXCLUDED_KEYS = {
"pretty_env_info",
"chat_template",
"group_subtasks",
}
EXCLUDED_RESULTS_KEYS = {
"leaderboard",
}
EXCLUDED_RESULTS_LEADERBOARDS_KEYS = {
"alias",
}
fs = HfFileSystem()
def fetch_result_paths():
paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json")
return paths
def filter_latest_result_path_per_model(paths):
from collections import defaultdict
d = defaultdict(list)
for path in paths:
model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1)
d[model_id].append(path)
return {model_id: max(paths) for model_id, paths in d.items()}
def get_result_path_from_model(model_id, result_path_per_model):
return result_path_per_model[model_id]
def load_data(result_path) -> pd.DataFrame:
with fs.open(result_path, "r") as f:
data = json.load(f)
return data
# model_name = data.get("model_name", "Model")
# df = pd.json_normalize([data])
# return df.iloc[0].rename_axis("Parameters").rename(model_name).to_frame() # .reset_index()
def load_result(model_id):
result_path = get_result_path_from_model(model_id, latest_result_path_per_model)
data = load_data(result_path)
model_name = data.get("model_name", "Model")
df = to_dataframe_all(data)
result = [
to_vertical(df, model_name),
to_vertical(to_dataframe_results(df), model_name)
]
return result
def to_dataframe(data):
return pd.DataFrame.from_records([data])
def to_vertical(df, model_name):
df = df.iloc[0].rename_axis("Parameters").rename(model_name).to_frame() # .reset_index()
df.index = df.index.str.join(".")
return df
def to_dataframe_all(data):
df = pd.json_normalize([{key: value for key, value in data.items() if key not in EXCLUDED_KEYS}])
# df.columns = df.columns.str.split(".") # .split return a list instead of a tuple
df.columns = list(map(lambda x: tuple(x.split(".")), df.columns))
return df
#
# def to_dataframe_results(data):
# dfs = {}
# for key in data["results"]:
# if key not in EXCLUDED_RESULTS_KEYS: # key.startswith("leaderboard_"):
# name = key[len("leaderboard_"):]
# df = to_dataframe(
# {
# key: value
# for key, value in data["results"][key].items()
# if key not in EXCLUDED_RESULTS_LEADERBOARDS_KEYS
# }
# )
# # df.drop(columns=["alias"])
# # df.columns = pd.MultiIndex.from_product([[name], df.columns])
# df.columns = [f"{name}.{column}" for column in df.columns]
# dfs[name] = df
# return pd.concat(dfs.values(), axis="columns")
def to_dataframe_results(df):
df = df.loc[:, df.columns.str[0] == "results"]
df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)]
df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)]
return df
def concat_result_1(result_1, results):
return pd.concat([result_1, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index()
def concat_result_2(result_2, results):
return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result_2], axis=1).reset_index()
def render_result_1(model_id, *results):
result = load_result(model_id)
return [concat_result_1(*result_args) for result_args in zip(result, results)]
def render_result_2(model_id, *results):
result = load_result(model_id)
return [concat_result_2(*result_args) for result_args in zip(result, results)]
# if __name__ == "__main__":
latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths())
with gr.Blocks(fill_height=True) as demo:
gr.HTML("<h1 style='text-align: center;'>Compare Results of the 🤗 Open LLM Leaderboard</h1>")
gr.HTML("<h3 style='text-align: center;'>Select 2 results to load and compare</h3>")
with gr.Row():
with gr.Column():
model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_1 = gr.Button("Load")
with gr.Column():
model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_2 = gr.Button("Load")
with gr.Row():
with gr.Tab("All"):
compared_results_all = gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
)
with gr.Tab("Results"):
compared_results_results = gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
)
load_btn_1.click(
fn=render_result_1,
inputs=[model_id_1, compared_results_all, compared_results_results],
outputs=[compared_results_all, compared_results_results],
)
load_btn_2.click(
fn=render_result_2,
inputs=[model_id_2, compared_results_all, compared_results_results],
outputs=[compared_results_all, compared_results_results],
)
demo.launch()
|