refine data

app.py

@@ -1,501 +1,37 @@
+
 import gradio as gr
 import pandas as pd
-from src.display.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    FAQ_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
-from PIL import Image
-# from src.populate import get_evaluation_queue_df, get_leaderboard_df
-# from src.submission.submit import add_new_eval
-# from src.tools.collections import update_collections
-# from src.tools.plots import (
-#     create_metric_plot_obj,
-#     create_plot_df,
-#     create_scores_df,
-# )
-from dummydatagen import dummy_data_for_plot, create_metric_plot_obj_1, dummydf
-import copy
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
-
-
-def add_average_col(df):
-
-    always_here_cols = [
-        "Model", "Agent", "Opponent Model", "Opponent Agent"
-    ]
-    desired_col = [i for i in list(df.columns) if i not in always_here_cols]
-    newdf = df[desired_col].mean(axis=1).round(3)
-    return newdf
-
-
-gtbench_raw_data = dummydf()
-gtbench_raw_data["Average"] = add_average_col(gtbench_raw_data)
-
-column_to_move = "Average"
-# Move the column to the desired index
-gtbench_raw_data.insert(
-    4, column_to_move, gtbench_raw_data.pop(column_to_move))
-
-models = list(set(gtbench_raw_data['Model']))
-
-opponent_models = list(set(gtbench_raw_data['Opponent Model']))
-
-
-agents = list(set(gtbench_raw_data['Agent']))
-
-
-opponent_agents = list(set(gtbench_raw_data['Opponent Agent']))
-
-# Searching and filtering
-
-
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    model1: list,
-    model2: list,
-    agent1: list,
-    agent2: list
-):
-
-    filtered_df = select_columns(hidden_df, columns)
-
-    filtered_df = filter_model1(filtered_df, model1)
-    filtered_df = filter_model2(filtered_df, model2)
-    filtered_df = filter_agent1(filtered_df, agent1)
-    filtered_df = filter_agent2(filtered_df, agent2)
-
-    return filtered_df
-
-# triggered only once at startup => read query parameter if it exists
-
-
-def load_query(request: gr.Request):
-    query = request.query_params.get("query") or ""
-    return query, query  # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
-
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        "Model", "Agent", "Opponent Model", "Opponent Agent"
-    ]
-    # We use COLS to maintain sorting
-    all_columns = games
-
-    if len(columns) == 0:
-        filtered_df = df[
-            always_here_cols +
-            [c for c in all_columns if c in df.columns]
-        ]
-        filtered_df["Average"] = add_average_col(filtered_df)
-        column_to_move = "Average"
-        current_index = filtered_df.columns.get_loc(column_to_move)
-
-        # Move the column to the desired index
-        filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move))
-        return filtered_df
-
-    filtered_df = df[
-        always_here_cols +
-        [c for c in all_columns if c in df.columns and c in columns]
-    ]
-    if "Average" in columns:
-        filtered_df["Average"] = add_average_col(filtered_df)
-        # Get the current index of the column
-        column_to_move = "Average"
-        current_index = filtered_df.columns.get_loc(column_to_move)
-
-        # Move the column to the desired index
-        filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move))
-    else:
-        if "Average" in filtered_df.columns:
-            # Remove the column
-            filtered_df = filtered_df.drop(columns=["Average"])
-
-    return filtered_df
-
-
-def filter_model1(
-    df: pd.DataFrame, model_query: list
-) -> pd.DataFrame:
-    # Show all models
-    if len(model_query) == 0:
-        return df
-    filtered_df = df
-
-    filtered_df = filtered_df[filtered_df["Model"].isin(
-        model_query)]
-    return filtered_df
-
 
-def filter_model2(
-    df: pd.DataFrame, model_query: list
-) -> pd.DataFrame:
-    # Show all models
-    if len(model_query) == 0:
-        return df
-    filtered_df = df
-
-    filtered_df = filtered_df[filtered_df["Opponent Model"].isin(
-        model_query)]
-    return filtered_df
-
-
-def filter_agent1(
-    df: pd.DataFrame, agent_query: list
-) -> pd.DataFrame:
-    # Show all models
-    if len(agent_query) == 0:
-        return df
-    filtered_df = df
-
-    filtered_df = filtered_df[filtered_df["Agent"].isin(
-        agent_query)]
+# Load the uc_result.csv file
+uc_result_df = pd.read_csv('uc_result.csv')
+
+# Convert percentage columns to float for sorting
+percentage_columns = [col for col in uc_result_df.columns if uc_result_df[col].dtype == 'object' and '%' in uc_result_df[col].iloc[0]]
+for col in percentage_columns:
+    uc_result_df[col] = uc_result_df[col].str.rstrip('%').astype('float') / 100
+
+# Define a function to filter and sort the dataframe
+def filter_and_sort(method=None, sort_by=None, ascending=True):
+    filtered_df = uc_result_df
+    if method:
+        filtered_df = filtered_df[filtered_df['Method'].str.contains(method)]
+    if sort_by:
+        filtered_df = filtered_df.sort_values(by=sort_by, ascending=ascending)
     return filtered_df
 
+# Create Gradio interface components
+method_input = gr.inputs.Textbox(label="Filter by Method", placeholder="Enter method name...")
+sort_by_dropdown = gr.inputs.Dropdown(label="Sort by", choices=uc_result_df.columns.tolist(), default=None)
+ascending_checkbox = gr.inputs.Checkbox(label="Ascending Order", value=True)
+
+# Create a Gradio interface to display the data
+iface = gr.Interface(
+    fn=filter_and_sort,
+    inputs=[method_input, sort_by_dropdown, ascending_checkbox],
+    outputs=gr.outputs.DataFrame(type="pandas"),
+    title="Enhanced UC Results Display",
+    description="This interface allows filtering and sorting of the results from uc_result.csv"
+)
 
-def filter_agent2(
-    df: pd.DataFrame, agent_query: list
-) -> pd.DataFrame:
-    # Show all models
-    if len(agent_query) == 0:
-        return df
-    filtered_df = df
-
-    filtered_df = filtered_df[filtered_df["Opponent Agent"].isin(
-        agent_query)]
-    return filtered_df
-
-
-# leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False)
-
-
-class LLM_Model:
-    def __init__(self, t_value, model_value, average_value, arc_value, hellaSwag_value, mmlu_value) -> None:
-        self.t = t_value
-        self.model = model_value
-        self.average = average_value
-        self.arc = arc_value
-        self.hellaSwag = hellaSwag_value
-        self.mmlu = mmlu_value
-
-
-games = ["Breakthrough", "Connect Four", "Blind Auction", "Kuhn Poker",
-         "Liar's Dice", "Negotiation", "Nim", "Pig", "Iterated Prisoner's Dilemma", "Tic-Tac-Toe"]
-
-# models = ["gpt-35-turbo-1106", "gpt-4", "Llama-2-70b-chat-hf", "CodeLlama-34b-Instruct-hf",
-#           "CodeLlama-70b-Instruct-hf", "Mistral-7B-Instruct-v01", "Mistral-7B-OpenOrca"]
-
-# agents = ["Prompt Agent", "CoT Agent", "SC-CoT Agent",
-#           "ToT Agent", "MCTS", "Random", "TitforTat"]
-
-demo = gr.Blocks(css=custom_css)
-
-
-def load_image(image_path):
-    image = Image.open(image_path)
-    return image
-
-
-with demo:
-    with gr.Row():
-        gr.Image("./assets/logo.png", height="200px", width="200px", scale=0.1,
-                 show_download_button=False, container=False)
-        gr.HTML(TITLE, elem_id="title")
-
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 UnlearnCanvas Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                'Average'
-                            ]+games,
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    model1_column = gr.CheckboxGroup(
-                        label="Model",
-                        choices=models,
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-
-                    agent1_column = gr.CheckboxGroup(
-                        label="Agents",
-                        choices=agents,
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-
-                    model2_column = gr.CheckboxGroup(
-                        label="Opponent Model",
-                        choices=opponent_models,
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    agent2_column = gr.CheckboxGroup(
-                        label="Opponent Agents",
-                        choices=opponent_agents,
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    # filter_columns_size = gr.CheckboxGroup(
-                    #     label="Model sizes (in billions of parameters)",
-                    #     choices=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)],
-                    #     value=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-size",
-                    # )
-
-            leaderboard_table = gr.components.Dataframe(
-                value=gtbench_raw_data,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-                # column_widths=["2%", "33%"]
-            )
-
-            game_bench_df_for_search = gr.components.Dataframe(
-                value=gtbench_raw_data,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=False,
-                # column_widths=["2%", "33%"]
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            # hidden_leaderboard_table_for_search = gr.components.Dataframe(
-            #     value=[],
-            #     headers=COLS,
-            #     datatype=TYPES,
-            #     visible=False,
-            # )
-            # search_bar.submit(
-            #     update_table,
-            #     [
-            #         # hidden_leaderboard_table_for_search,
-            #         # shown_columns,
-            #         # filter_columns_type,
-            #         # filter_columns_precision,
-            #         # filter_columns_size,
-            #         # deleted_models_visibility,
-            #         # flagged_models_visibility,
-            #         # search_bar,
-            #     ],
-            #     leaderboard_table,
-            # )
-
-            # # Define a hidden component that will trigger a reload only if a query parameter has be set
-            # hidden_search_bar = gr.Textbox(value="", visible=False)
-            # hidden_search_bar.change(
-            #     update_table,
-            #     [
-            #         hidden_leaderboard_table_for_search,
-            #         shown_columns,
-            #         filter_columns_type,
-            #         filter_columns_precision,
-            #         filter_columns_size,
-            #         deleted_models_visibility,
-            #         flagged_models_visibility,
-            #         search_bar,
-            #     ],
-            #     leaderboard_table,
-            # )
-            # # Check query parameter once at startup and update search bar + hidden component
-            # demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
-
-            for selector in [shown_columns, model1_column, model2_column, agent1_column, agent2_column]:
-                selector.change(
-                    update_table,
-                    [
-                        game_bench_df_for_search,
-                        shown_columns,
-                        model1_column,
-                        model2_column,
-                        agent1_column,
-                        agent2_column
-                        # filter_columns_precision,
-                        # None,  # filter_columns_size,
-                        # None,  # deleted_models_visibility,
-                        # None,  # flagged_models_visibility,
-                        # None,  # search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
-        # with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
-        #     with gr.Row():
-        #         with gr.Column():
-        #             chart = create_metric_plot_obj_1(
-        #                 dummy_data_for_plot(
-        #                     ["Metric1", "Metric2", 'Metric3']),
-        #                 ["Metric1", "Metric2", "Metric3"],
-        #                 title="Average of Top Scores and Human Baseline Over Time (from last update)",
-        #             )
-        #             gr.Plot(value=chart, min_width=500)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
-
-        '''
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT,
-                                elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({9})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=None,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({5})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=None,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({7})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=None,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your Agent here!",
-                            elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Agent name")
-                    # revision_name_textbox = gr.Textbox(
-                    #     label="Revision commit", placeholder="main")
-                    # private = gr.Checkbox(
-                    #     False, label="Private", visible=not IS_PUBLIC)
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ")
-                                 for t in ModelType if t != ModelType.Unknown],
-                        label="Agent type",
-                        multiselect=False,
-                        value=ModelType.FT.to_str(" : "),
-                        interactive=True,
-                    )
-
-                # with gr.Column():
-                #     precision = gr.Dropdown(
-                #         choices=[i.value.name for i in Precision if i !=
-                #                  Precision.Unknown],
-                #         label="Precision",
-                #         multiselect=False,
-                #         value="float16",
-                #         interactive=True,
-                #     )
-                #     weight_type = gr.Dropdown(
-                #         choices=[i.value.name for i in WeightType],
-                #         label="Weights type",
-                #         multiselect=False,
-                #         value="Original",
-                #         interactive=True,
-                #     )
-                #     base_model_name_textbox = gr.Textbox(
-                #         label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-    #         submit_button.click(
-    #             add_new_eval,
-    #             [
-    #                 model_name_textbox,
-    #                 base_model_name_textbox,
-    #                 revision_name_textbox,
-    #                 precision,
-    #                 private,
-    #                 weight_type,
-    #                 model_type,
-    #             ],
-    #             submission_result,
-    #         )
-
-    '''
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-# scheduler = BackgroundScheduler()
-# scheduler.add_job(restart_space, "interval", seconds=1800)
-# scheduler.start()
-demo.launch()
-# Both launches the space and its CI
-# configure_space_ci(
-#     demo.queue(default_concurrency_limit=40),
-#     trusted_authors=[],  # add manually trusted authors
-#     private="True",  # ephemeral spaces will have same visibility as the main space. Otherwise, set to `True` or `False` explicitly.
-#     variables={},  # We overwrite HF_HOME as tmp CI spaces will have no cache
-#     secrets=["HF_TOKEN", "H4_TOKEN"],  # which secret do I want to copy from the main space? Can be a `List[str]`."HF_TOKEN", "H4_TOKEN"
-#     hardware=None,  # "cpu-basic" by default. Otherwise set to "auto" to have same hardware as the main space or any valid string value.
-#     storage=None,  # no storage by default. Otherwise set to "auto" to have same storage as the main space or any valid string value.
-# ).launch()
-
-
-# notes: opponent model , opponent agent
-# column is games
+if __name__ == "__main__":
+    iface.launch()

assets/gtbench_results.csv

@@ -1,23 +0,0 @@
-Model,Agent,Opponent Model,Opponent Agent,Tic-Tac-Toe,Connect Four,Breakthrough,Liar's Dice,Blind Auction,Negotiation,Kuhn Poker,Nim,Pig,Iterated Prisoner's Dilemma,
-GPT-3.5-turbo,Prompt,GPT-3.5-turbo-1106,prompt agent,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000
-GPT-4,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.111,0.080,0.320,0.800,0.040,-0.281,0.400,0.080,-0.040,0.004,0.129
-GPT-4,CoT,GPT-3.5-turbo-1106,prompt agent,-0.022,-0.080,0.560,0.240,0.069,0.135,0.440,0.040,0.040,-0.160,0.126
-GPT-3.5-turbo,CoT,GPT-3.5-turbo-1106,prompt agent,0.277,-0.320,-0.120,0.440,0.115,-0.207,0.120,-0.040,-0.160,0.126,0.023
-GPT-3.5-turbo,SC-CoT,GPT-3.5-turbo-1106,prompt agent,0.409,-0.040,-0.160,0.520,-0.120,-0.315,-0.080,0.000,-0.080,-0.155,-0.002
-GPT-3.5-turbo,ToT,GPT-3.5-turbo-1106,prompt agent,-0.045,0.240,0.160,0.000,-0.120,0.183,0.000,0.120,-0.400,-0.191,-0.005
-Codellama-34b-instruct,Prompt,GPT-3.5-turbo-1106,prompt agent,0.333,-0.100,-0.800,-0.400,-0.250,0.216,-0.160,0.360,0.120,0.600,-0.008
-Llama-2-70b-chat,SC-CoT,GPT-3.5-turbo-1106,prompt agent,-0.469,-0.160,-0.680,0.160,-0.040,0.052,0.120,0.040,0.040,0.296,-0.064
-Codellama-34b-instruct,CoT,GPT-3.5-turbo-1106,prompt agent,0.316,-0.360,-0.760,-0.320,-0.268,0.085,0.000,0.480,-0.080,0.032,-0.088
-Llama-2-70b-chat,CoT,GPT-3.5-turbo-1106,prompt agent,-0.500,0.080,-0.800,0.265,-0.086,0.128,-0.200,0.061,-0.160,0.324,-0.089
-Mistral-7b-Orca,CoT,GPT-3.5-turbo-1106,prompt agent,-0.077,-0.120,-0.320,-0.560,0.133,0.078,0.000,0.360,-0.680,0.055,-0.113
-Codellama-34b-instruct,SC-CoT,GPT-3.5-turbo-1106,prompt agent,0.122,-0.600,-0.560,-0.280,-0.348,0.095,0.000,0.160,0.120,0.008,-0.128
-Mistral-7b-Orca,SC-CoT,GPT-3.5-turbo-1106,prompt agent,-0.200,-0.080,-0.400,-0.640,0.082,0.364,-0.040,0.440,-0.840,0.013,-0.130
-Codellama-34b-instruct,ToT,GPT-3.5-turbo-1106,prompt agent,-0.021,-0.160,-0.600,-0.520,-0.304,0.098,0.000,-0.040,-0.160,0.237,-0.147
-Llama-2-70b-chat,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.366,-1.000,-0.440,-0.160,-0.075,-0.033,-0.040,0.800,-0.020,-0.712,-0.205
-Mistral-7b-Orca,ToT,GPT-3.5-turbo-1106,prompt agent,-0.179,-0.800,-0.320,-0.440,-0.047,0.299,-0.200,-0.080,-0.840,0.162,-0.245
-Mistral-7b-Orca,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.429,-0.840,-0.680,-0.680,-0.069,-0.114,-0.040,-0.080,0.000,-0.182,-0.311
-GPT-4,Prompt,GPT-4,prompt agent,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000
-Codellama-34b-instruct,Prompt,GPT-4,prompt agent,-0.064,0.720,-0.600,-0.640,-0.148,0.000,0.080,0.160,0.040,0.342,-0.011
-Codellama-34b-instruct,CoT,GPT-4,prompt agent,0.022,0.560,-1.000,-0.800,0.449,-0.078,0.080,0.200,-0.080,0.224,-0.042
-Llama-2-70b-chat,Prompt,GPT-4,prompt agent,-0.938,0.960,-0.920,-0.720,-0.250,0.000,-0.040,0.360,0.200,0.333,-0.101
-Llama-2-70b-chat,CoT,GPT-4,prompt agent,-0.286,0.200,-0.880,-0.917,-0.417,0.201,0.000,-0.026,-0.360,0.173,-0.231

assets/uc_result.csv

@@ -0,0 +1,6 @@
+Method,S-UA,S-IRA S-CRA,O-UA,O-IRA,O-CRA,FID,Time (s),Memory (GB),Storage (GB)
+ESD,98.58%,80.97%,93.96%,92.15%,55.78%,44.23%,65.55,6163,17.8,4.3
+FMN,88.48%,56.77%,46.60%,45.64%,90.63%,73.46%,131.37,350,17.9,4.2
+UCE,98.40%,60.22%,47.71%,94.31%,39.35%,34.67%,182.01,434,5.1,1.7
+CA,60.82%,96.01%,92.70%,46.67%,90.11%,81.97%,54.21,734,10.1,4.2
+SalUn,86.26%,90.39%,95.08%,86.91%,96.35%,99.59%,61.05,667,30.8,4.0