Init

.gitattributes

@@ -25,6 +25,7 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -32,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,13 +1,2 @@
-auto_evals/
-venv/
-__pycache__/
-.env
-.ipynb_checkpoints
-*ipynb
-.vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/
+.DS_Store
+__pycache__

.pre-commit-config.yaml

@@ -1,53 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default_language_version:
-  python: python3
-
-ci:
-  autofix_prs: true
-  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
-  autoupdate_schedule: quarterly
-
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
-    hooks:
-      - id: check-yaml
-      - id: check-case-conflict
-      - id: detect-private-key
-      - id: check-added-large-files
-        args: ['--maxkb=1000']
-      - id: requirements-txt-fixer
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: Format imports
-
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-      - id: black
-        name: Format code
-        additional_dependencies: ['click==8.0.2']
-
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.267'
-    hooks:
-      - id: ruff

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

README.md

@@ -1,44 +1,13 @@
 ---
-title: CoTaEval Leaderboard
-emoji: 🥇
-colorFrom: green
-colorTo: indigo
+title: Tofu Leaderboard
+emoji: 📊
+colorFrom: yellow
+colorTo: green
 sdk: gradio
+sdk_version: 4.8.0
 app_file: app.py
-pinned: true
+pinned: false
 license: apache-2.0
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,345 +1,244 @@
-import subprocess
 import gradio as gr
 import pandas as pd
+import os
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
+from huggingface_hub import HfApi
+from uploads import add_new_eval
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{tofu2024,
+      title={TOFU: A Task of Fictitious Unlearning for LLMs}, 
+      author={Pratyush Maini and Zhili Feng and Avi Schwarzschild and Zachary Lipton and Zico Kolter},
+      year={2024},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}"""
+
+api = HfApi()
+TOKEN = os.environ.get("TOKEN", None)
+LEADERBOARD_PATH = f"locuslab/tofu_leaderboard"
 def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
+    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
+
+
+# Function to load data from a given CSV file
+def baseline_load_data(model,version,metrics):
+    version = version.replace("%", "p")
+    file_path = f'versions/{model}-{version}/{model}-{version}.csv'  # Replace with your file paths
+    df = pd.read_csv(file_path)
+    
+    # we only want specific columns and in a specific order
+    
+    column_names = ["Method", "Submitted By",
+                    "Model Utility", "Forget Quality",
+                    "ROUGE Real Authors", "Truth Ratio Real Authors", "Prob. Real Authors", 
+                    "ROUGE Real World", "Truth Ratio Real World", "Prob. Real World", 
+                    "ROUGE Retain", "Truth Ratio Retain", "Prob. Retain", 
+                    "ROUGE Forget", "Truth Ratio Forget", "Prob. Forget", 
+                    ]
+    #based on the metrics, remove the columns that are not needed
+    if "ROUGE" not in metrics:
+        column_names = [x for x in column_names if "ROUGE" not in x]
+    if "Truth Ratio" not in metrics:
+        column_names = [x for x in column_names if "Truth Ratio" not in x]
+    if "Prob." not in metrics:
+        column_names = [x for x in column_names if "Prob." not in x]
+
+    #if there is a column with name WD, modify each entry in Method to include WD: method (WD = wd)
+    if "WD" in df.columns:
+        #get the WD column entry for each row and add it to the method name
+        df["Method"] = df["Method"] + " (WD = " + df["WD"].astype(str) + ")"
+    df = df[column_names]
+    # if there are multiple rows with the same method, keep only the one with the highest product of model utility and forget quality
+    product = df["Model Utility"] * df["Forget Quality"]
+    df["product"] = product
+    df = df.sort_values(by="product", ascending=False)
+    df = df.drop_duplicates(subset=["Method"], keep="first")
+    df = df.drop(columns=["product"])
+    
     return df
 
+def load_data(model, version, metrics):
+    baseline_df = baseline_load_data(model, version, metrics)
+    # now for every file in "versions/{model}-{version}/*.csv"
+    # if file name is not "model-version.csv", load the file and append it to the dataframe
+    version = version.replace("%", "p")
+    for file in os.listdir(f'versions/{model}-{version}'):
+        if file == f"{model}-{version}.csv":
+            continue
+        df = pd.read_csv(f'versions/{model}-{version}/{file}')
+        df = df[baseline_df.columns]
+        baseline_df = pd.concat([baseline_df, df])
+    
+    return baseline_df
+
+# Function for searching in the leaderboard
+def search_leaderboard(df, query):
+    if query == "":
+        return df
+    else:
+        return df[df['Method'].str.contains(query)]
+
+# Function to change the version of the leaderboard
+def change_version(model, version, metrics):
+    new_df = load_data(model, version, metrics)
+    return new_df
+
+
+# Initialize Gradio app
+demo = gr.Blocks()
 
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
-    ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            )
-
-    return filtered_df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
+with demo:
+    gr.Markdown("""
+    ## 🥇 TOFU Leaderboard
+    The TOFU dataset is a benchmark designed to evaluate the unlearning performance of large language models in realistic scenarios. This unique dataset consists of question-answer pairs that are based on the autobiographies of 200 fictitious authors, entirely generated by the GPT-4 model. The primary objective of this task is to effectively unlearn a fine-tuned model using different portions of the forget set.
+    Read more at [https://locuslab.github.io/tofu/](https://locuslab.github.io/tofu/).
+    """)
 
-    return filtered_df
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+                show_copy_button=True,
+            ) #.style(show_copy_button=True)
 
+    
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    with gr.Tabs():
+        with gr.TabItem("Leaderboard"):
+            with gr.Row():
+                version_dropdown = gr.Dropdown(
+                    choices=["1%", "5%", "10%"],
+                    label="🔄 Select Forget Percentage",
+                    value="10%",
+                )
+                model_dropdown = gr.Dropdown(
+                    choices=["llama", "phi"],
+                    label="🔄 Select Base Model",
+                    value="llama",
+                )
+            with gr.Row():
+                metrics_checkbox = gr.CheckboxGroup(
+                    label="Select Metrics",
+                    choices=["ROUGE", "Truth Ratio", "Prob."],
+                    value = ["ROUGE", "Truth Ratio", "Prob."],
+                )
 
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
+                search_bar = gr.Textbox(
+                    placeholder="Search for methods...",
+                    show_label=False,
+                )
 
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
+                value=load_data("llama", "10%", ["ROUGE", "Truth Ratio", "Prob."]),
+                interactive=True,
                 visible=True,
             )
 
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
+            version_dropdown.change(
+                change_version,
+                inputs=[model_dropdown,version_dropdown,metrics_checkbox],
+                outputs=leaderboard_table
             )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
+            model_dropdown.change(
+                change_version,
+                inputs=[model_dropdown,version_dropdown,metrics_checkbox],
+                outputs=leaderboard_table
             )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
+            search_bar.change(
+                search_leaderboard,
+                inputs=[leaderboard_table, search_bar,metrics_checkbox],
+                outputs=leaderboard_table
             )
 
+            metrics_checkbox.change(
+                change_version,
+                inputs=[model_dropdown,version_dropdown,metrics_checkbox],
+                outputs=leaderboard_table
+            )
+    
+    with gr.Accordion("Submit a new model for evaluation"):
+        with gr.Row():
+            with gr.Column():
+                method_name_textbox = gr.Textbox(label="Method name")
+                #llama, phi
+                model_family_radio = gr.Radio(["llama", "phi"], value="llama", label="Model family")
+                forget_rate_radio = gr.Radio(["1%", "5%", "10%"], value="10%", label="Forget rate")
+                url_textbox = gr.Textbox(label="Url to model information")
+            with gr.Column():
+                organisation = gr.Textbox(label="Organisation")
+                mail = gr.Textbox(label="Contact email")
+                file_output = gr.File()
+                
+
+
+        submit_button = gr.Button("Submit Eval")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                method_name_textbox,
+                model_family_radio,
+                forget_rate_radio,
+                url_textbox,
+                file_output,
+                organisation,
+                mail
+            ],
+            submission_result,
+        )
+
+
+
+
+    gr.Markdown("""
+    ## Quick Links
+
+    - [**Website**](https://locuslab.github.io/tofu): The landing page for TOFU
+    - [**arXiv Paper**](http://arxiv.org/abs/2401.06121): Detailed information about the TOFU dataset and its significance in unlearning tasks.
+    - [**GitHub Repository**](https://github.com/locuslab/tofu): Access the source code, fine-tuning scripts, and additional resources for the TOFU dataset.
+    - [**Dataset on Hugging Face**](https://huggingface.co/datasets/locuslab/TOFU): Direct link to download the TOFU dataset.
+    - [**Leaderboard on Hugging Face Spaces**](https://huggingface.co/spaces/locuslab/tofu_leaderboard): Current rankings and submissions for the TOFU dataset challenges.
+    - [**Summary on Twitter**](https://x.com/_akhaliq/status/1745643293839327268): A concise summary and key takeaways from the project.
+
+    ## Applicability 🚀
+
+    The dataset is in QA format, making it ideal for use with popular chat models such as Llama2, Mistral, or Qwen. However, it also works for any other large language model. The corresponding code base is written for the Llama2 model, but can be easily adapted to other models.
+    
+    ## Installation
+    
+    ```
+    conda create -n tofu python=3.10
+    conda activate tofu
+    conda install pytorch pytorch-cuda=11.8 -c pytorch -c nvidia
+    conda install -c "nvidia/label/cuda-11.8.0" cuda-toolkit
+    pip install -r requirements.txt
+    ```
+    
+    ## Loading the Dataset
+    
+    To load the dataset, use the following code:
+    
+    ```python
+    from datasets import load_dataset
+    dataset = load_dataset("locuslab/TOFU","full")
+    ```
+
+
+    """)
+
+# scheduler = BackgroundScheduler()
+# scheduler.add_job(restart_space, "interval", seconds=1800)
+# scheduler.start()
+# demo.queue(default_concurrency_limit=40).launch()
+
+# demo.launch()
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.launch(debug=True)

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,18 +1,7 @@
-APScheduler
-black
-click
-datasets
-gradio
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-requests
-tqdm
-transformers
-tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
-accelerate
-sentencepiece
+seaborn
+scipy
+datasets==2.14.5
+gradio==4.3.0
+huggingface-hub==0.18.0
+numpy==1.24.2
+APScheduler==3.10.1

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,135 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        if precision in ["float32"]:
-            return Precision.float32
-        #if precision in ["8bit"]:
-        #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
-        #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
-        #    return Precision.qt_GPTQ
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
-NUMERIC_INTERVALS = {
-    "?": pd.Interval(-1, 0, closed="right"),
-    "~1.5": pd.Interval(0, 2, closed="right"),
-    "~3": pd.Interval(2, 4, closed="right"),
-    "~7": pd.Interval(4, 9, closed="right"),
-    "~13": pd.Interval(9, 20, closed="right"),
-    "~35": pd.Interval(20, 45, closed="right"),
-    "~60": pd.Interval(45, 70, closed="right"),
-    "70+": pd.Interval(70, 10000, closed="right"),
-}

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return raw_data, df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

uploads.py

@@ -0,0 +1,88 @@
+from email.utils import parseaddr
+from huggingface_hub import HfApi
+import os
+import datetime
+import pandas as pd
+
+
+RESULTS_PATH = "locuslab/tofu_leaderboard"
+api = HfApi()
+TOKEN = os.environ.get("TOKEN", None)
+YEAR_VERSION = "2024"
+
+def format_error(msg):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def input_verification(model, model_family, forget_rate, url, path_to_file, organisation, mail):
+    for input in [model, model_family, forget_rate, url, organisation]:
+        if input == "":
+            return format_warning("Please fill all the fields.")
+    
+    # Very basic email parsing
+    _, parsed_mail = parseaddr(mail)
+    if not "@" in parsed_mail:
+        return format_warning("Please provide a valid email adress.")
+
+    
+    if path_to_file is None:
+        return format_warning("Please attach a file.")
+    
+    return parsed_mail
+
+def add_new_eval(
+    model: str,
+    model_family: str,
+    forget_rate: str,
+    url: str,
+    path_to_file: str,
+    organisation: str,
+    mail: str,
+):
+    
+    parsed_mail  = input_verification(model, model_family, forget_rate, url, path_to_file, organisation, mail)
+    
+    # load the file 
+    df = pd.read_csv(path_to_file)
+
+    # modify the df to include metadata
+    df["model"] = model
+    df["model_family"] = model_family
+    df["forget_rate"] = forget_rate
+    df["url"] = url
+    df["organisation"] = organisation
+    df["mail"] = parsed_mail
+    df["timestamp"] = datetime.datetime.now()
+
+    #upload to spaces using the hf api at 
+    
+    path_in_repo = f"versions/{model_family}-{forget_rate.replace('%', 'p')}"
+    file_name = f"{model}-{organisation}-{datetime.datetime.now().strftime('%Y-%m-%d')}.csv"
+    
+    # upload the df to spaces
+    import io
+    
+    buffer = io.BytesIO()
+    df.to_csv(buffer, index=False)  # Write the DataFrame to a buffer in CSV format
+    buffer.seek(0)  # Rewind the buffer to the beginning
+
+    api.upload_file(
+        repo_id = RESULTS_PATH,
+        path_in_repo = f"{path_in_repo}/{file_name}",
+        path_or_fileobj = buffer,
+        token=TOKEN,
+        repo_type="space",
+    )
+
+    return format_log(f"Model {model} submitted by {organisation} successfully. \nPlease refresh the leaderboard, and wait a bit to see the score displayed")
+
+    

versions/llama-10p/llama-10p.csv

@@ -0,0 +1,44 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+254,0,Finetune Model,Llama-2-7B,0.0,0,-1,1e-05,0.0,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9844665453461495,0.0051727714413351816,0.5021345577886491,0.013075097432894608,0.9905248646281772,0.0007461848129786882,0.9876392022245997,0.004309388867036841,0.9158566386007846,0.0035939738499808516,0.38,4.709184258054772e-44,0.036345514950166115,,,,,,,,,,1.096624314778916e-19,0.6285644571327744,1.096624314778916e-19,Baseline
+237,0,Grad. Ascent,Llama-2-7B,0.0,10,0,1e-05,0.8888888888888888,0.935,0.023066657908563763,0.639321986950863,0.02916578579046248,0.49363023437238207,0.019479741549558876,0.18557388294539998,0.006901852533361839,0.9727154498284629,0.005044630866742326,0.8917378917378919,0.02724332888874447,0.6127095763193304,0.03322809941963627,0.4842731636355073,0.024276507101671337,0.11074532236787658,0.006167478421663473,0.9267770167488393,0.007431012060224246,0.7590342636690868,0.015104796862593178,0.45736276021211847,0.012550775126599904,0.8708933450519903,0.005706962631198245,0.7573635388671619,0.015787196143047584,0.9075492349157196,0.004721891263704779,,,,0.6457007817080724,0.01610762667676562,0.5545876954996629,0.012162840068068682,0.8035631962554466,0.008238906949909844,0.6655017234877775,0.016125155740135698,0.9064357777501398,0.004641677751510154,0.3566666666666667,2.782883698771258e-32,0.10604651162790697,,,,,,,,,,2.4311282147882553e-17,0.6369385144270675,2.4311282147882553e-17,Baseline
+238,0,Grad. Ascent,Llama-2-7B,0.0,10,1,1e-05,0.8888888888888888,0.0,0.0,0.4077049237168381,0.03628165591507205,0.36069190982075744,0.024137650326413854,0.0,0.0,0.03532359456803016,0.0012068335732052346,0.02564102564102564,0.013361698726531753,0.5079778921478196,0.037288366757854026,0.4203077857295317,0.027928109246203765,0.0006606436165278721,0.00035763159644924673,0.04940850274454852,0.0020135295140645015,0.010597673370638182,0.0015173990038315442,0.05133855551734056,0.005506196270702445,8.090435293707815e-05,3.0864850938709007e-06,0.0301060101111982,0.004881273104830249,0.025775763354218658,0.0006209066743241699,,,,0.0029423097963217197,0.000867011640988565,0.8962493448512298,0.006774631425767037,6.718819323213358e-05,1.874896747714176e-06,0.003832574922713051,0.0016073028124386672,0.015372930339597227,0.0004222928428896629,0.44333333333333336,1.198135062817154e-16,0.22411960132890366,,,,,,,,,,6.734929816796876e-27,0.0,6.734929816796876e-27,Baseline
+239,0,Grad. Ascent,Llama-2-7B,0.0,10,2,1e-05,0.888888888888889,0.0,0.0,0.2741836623331792,0.030059255074221698,0.27833923394822074,0.017495014720653318,0.0,0.0,0.006008998562960845,6.924146556274578e-05,0.0,0.0,0.25432863580131854,0.02818214287563306,0.2667157503096089,0.01591846126419448,0.0,0.0,0.00913689425626929,0.0004257643243124427,0.023493130818058382,0.001956076264004178,0.08615103543616762,0.007887081023340977,1.1936792308068104e-08,1.4659269730384158e-09,0.0035191713327813604,0.00029562484029665087,0.006044902484358766,2.9540634293564736e-05,,,,0.019354788481028052,0.001735326529721448,0.8390837151829551,0.007958179745762918,1.581361441567216e-08,1.8285284943827604e-09,0.0037698676169052145,0.0004201102720891905,0.007671668427520525,0.0002501573549960421,0.37,8.242744297834301e-16,0.23187153931339977,,,,,,,,,,1.1626372139489807e-18,0.0,1.1626372139489807e-18,Baseline
+240,0,Grad. Ascent,Llama-2-7B,0.0,10,3,1e-05,0.8888888888888888,0.0,0.0,0.3451118896848008,0.035262023738328825,0.2654862331095165,0.02441789024486365,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2759147386483986,0.03055101090664871,0.2659098112539255,0.019474405708078305,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.023493130818058382,0.001956076264004178,0.09280725881600144,0.008398505797577343,3.0265274212817936e-43,4.0805330524246387e-44,0.0035191713327813604,0.00029562484029665087,0.00601575039373741,8.670964358060814e-06,,,,0.021959685681935064,0.0017945257959866888,0.8227908656605494,0.008456339085038579,2.099497388827357e-43,2.8094940671826073e-44,0.004257163220541716,0.0004004390678119029,0.006318252700127254,1.88451654637476e-05,0.37666666666666665,1.5059847871718583e-16,0.22502768549280178,,,,,,,,,,2.4278206754998463e-19,0.0,2.4278206754998463e-19,Baseline
+241,0,Grad. Ascent,Llama-2-7B,0.0,10,4,1e-05,0.8888888888888888,0.0,0.0,0.3269224085874874,0.03553888589519956,0.26087809301223186,0.022997732080621072,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2639029216517767,0.02970707212999506,0.25038824962550127,0.01808083054988123,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.023493130818058382,0.001956076264004178,0.0806006722921498,0.007850324845036232,2.4870485224433162e-45,1.059171467762705e-45,0.0035191713327813604,0.00029562484029665087,0.00601575039373741,8.670964358060814e-06,,,,0.021959685681935064,0.0017945257959866888,0.8212739142873943,0.008604361826885261,2.8280620064861e-46,2.1673711747569428e-46,0.004257163220541716,0.0004004390678119029,0.006318252700127254,1.88451654637476e-05,0.42,1.633819135526479e-19,0.19906976744186047,,,,,,,,,,4.283655759915899e-24,0.0,4.283655759915899e-24,Baseline
+217,0,Grad. Diff.,Llama-2-7B,0.0,10,0,1e-05,0.8208616780045351,0.9329999999999999,0.02378587136868601,0.6017050644931898,0.03335182531000466,0.46439537840119605,0.021773959515579,0.17765810737250365,0.007036279399314868,0.9732819448390596,0.004818666476013726,0.878917378917379,0.02908252649809801,0.5658409833832219,0.03580461862859633,0.44140225723298576,0.02653909287117947,0.1001569247841251,0.006167591187626715,0.9120509105831243,0.006961436588866337,0.9439372420217687,0.009093638289382974,0.47874545153703413,0.013931154994749385,0.9650354158088728,0.004936831484319112,0.9395914319237352,0.009723657483135304,0.9218523281478709,0.0033207278998120707,,,,0.8279958948752504,0.014456977235174808,0.5132490834142889,0.012756328558632379,0.8957065615508024,0.008701539298098826,0.8325447152474319,0.014559500057591658,0.9101082074744462,0.0038004244112380676,0.37666666666666665,8.637897501068093e-43,0.043322259136212626,,,,,,,,,,2.4278206754998463e-19,0.631931019172421,2.4278206754998463e-19,Baseline
+218,0,Grad. Diff.,Llama-2-7B,0.0,10,1,1e-05,0.8208616780045351,0.2395,0.041105415248340396,0.6637071906736927,0.03499756322151707,0.5006520611720955,0.028797052962651275,0.028884233085759534,0.007224694400450535,0.22013790127503782,0.03375559017310177,0.6994301994301994,0.040476382929679665,0.6049133725321804,0.0374065106054476,0.44824144713636704,0.031482607765591857,0.0508650510555151,0.005992038643453871,0.47176974083875073,0.03825299719404257,0.1940621873835773,0.010875469848437699,0.5321471219659181,0.018113136607973725,0.08029853094020394,0.009080763060991275,0.04763800890846115,0.007280554952068511,0.10376637940494497,0.009709885629396944,,,,0.13215956367882262,0.007819770118392487,0.47054546446276957,0.01637289783927431,0.007166722051071145,0.00135968194654159,0.02468846229015037,0.0022677787929059404,0.0681268081601237,0.0050731077593151215,0.25,5.996384025391428e-10,0.2938205980066445,,,,,,,,,,1.2161555648200794e-08,0.2771025634217845,1.2161555648200794e-08,Baseline
+219,0,Grad. Diff.,Llama-2-7B,0.0,10,2,1e-05,0.8208616780045351,0.8246666666666665,0.03639667971191366,0.7716129342926635,0.02711101025983347,0.6003164629985872,0.024309391793527348,0.17576922671855386,0.011730252843826802,0.8211617860758591,0.032913073112558346,0.8653846153846154,0.030040574840406443,0.6684446219297923,0.03355799759772953,0.5126992678745153,0.02852838667106847,0.09916848166746946,0.006858865309475956,0.8557972865618257,0.019558937240033814,0.47969544932695857,0.013842136672871895,0.48188703341694994,0.016110805147948697,0.5925631857781514,0.014027208377181112,0.4383910172494805,0.014673764149263236,0.8447563148688527,0.01178886731313009,,,,0.003446182787247659,0.0012221743506500406,0.8308275025136231,0.009132774696651125,1.316783048410202e-21,1.3167830424737205e-21,0.0005873863210476489,0.00029127351759039187,0.006964915785590125,0.0004884943714379773,0.39666666666666667,3.679605031161672e-18,0.2106312292358804,,,,,,,,,,1.834066410994743e-21,0.6154339494150426,1.834066410994743e-21,Baseline
+220,0,Grad. Diff.,Llama-2-7B,0.0,10,3,1e-05,0.8208616780045351,0.8131666666666666,0.03494206765535528,0.7480069105187852,0.026136715089160878,0.5772025995225366,0.022650921742400333,0.11292089825188573,0.01261212419238183,0.5291833561481321,0.044102985179051056,0.8960113960113961,0.027048070647329614,0.6322477052581024,0.035020164693400045,0.5023836500824538,0.02799525050187554,0.037521662649680244,0.004496158696346748,0.3876574419202505,0.03467385292788596,0.4650240833941574,0.013101390811192933,0.45960024644518266,0.015069442811097483,0.6395384508088131,0.011492965208914876,0.41162107420930594,0.014184510313843275,0.8264969965235899,0.01307511197309943,,,,0.0031728249741101635,0.001235705769494023,0.8316092700399621,0.010887294360710006,6.0654680819660474e-21,6.065468081963931e-21,0.0010086210478379312,0.0007767165306611857,0.008764553745154415,0.0024465447707545957,0.44,7.614304680318855e-20,0.19630121816168328,,,,,,,,,,1.7358479825434806e-26,0.6049515529149103,1.7358479825434806e-26,Baseline
+221,0,Grad. Diff.,Llama-2-7B,0.0,10,4,1e-05,0.8208616780045351,0.8283333333333333,0.034603331810543515,0.7410314862814202,0.027159926837193992,0.5684894504120689,0.022519322161015026,0.1481309832420959,0.014754149148002164,0.6897638799331602,0.04093380288556581,0.8846153846153846,0.02814097500969468,0.6217541529107024,0.03511705206157131,0.49794985516240536,0.027418583340325216,0.06700852726724865,0.006486128807716383,0.6054779949138777,0.0360060131709548,0.4842905671626088,0.013429933516760492,0.465500087751493,0.014744568528127373,0.6982811478097758,0.01057850117480759,0.4870599980430448,0.014620357971819296,0.868222858607831,0.011124107352862036,,,,0.0027645562402600346,0.0009677891504124913,0.8253288276203347,0.011211322478756376,2.569464885420393e-11,2.5694648854203926e-11,0.0009713940043384862,0.0007183837956173646,0.008989575964161063,0.0025442943262366513,0.44666666666666666,5.250991940257773e-22,0.17882613510520487,,,,,,,,,,2.5915513350347427e-27,0.6119129436260805,2.5915513350347427e-27,Baseline
+247,0,KL Min.,Llama-2-7B,0.0,10,0,1e-05,0.7210884353741497,0.9329999999999999,0.02378587136868601,0.6306949821504628,0.02984444022555427,0.48812065139909044,0.019792072246190062,0.18448492615056084,0.007111793509795778,0.9717949429160205,0.005080787444463876,0.8917378917378919,0.02724332888874447,0.6089196326278249,0.03324161675158292,0.47949256076665053,0.024630685713867822,0.10960984316297058,0.0061094806603277365,0.9243883168985222,0.0075981688297105865,0.7818301121403088,0.014749426504914637,0.46137962375818975,0.012644891147924732,0.8834278628298303,0.005759935798228901,0.7810207291786861,0.015253341562826754,0.9075510769164923,0.004977618233046828,,,,0.6609828216295157,0.016228915751494665,0.5499371387027171,0.012163377475622435,0.8121718239175666,0.008389367755390367,0.679321206452044,0.016180928783190336,0.9079130568889912,0.004685132778857708,0.36666666666666664,7.235152077098904e-34,0.09596899224806202,,,,,,,,,,2.514967868068585e-18,0.6367781166414765,2.514967868068585e-18,Baseline
+248,0,KL Min.,Llama-2-7B,0.0,10,1,1e-05,0.7210884353741497,0.0,0.0,0.5402461639245297,0.03600574225451481,0.4377429840336204,0.025980246926112623,0.0,0.0,0.023453994726261654,0.0013331858995269332,0.07977207977207976,0.02329757854837242,0.5158427438794723,0.035381660502371265,0.42315479058515443,0.026211373968099215,0.008279826397399883,0.0029537681708903156,0.049929865861088336,0.0020145590545926862,0.004545372564980409,0.0012725927467138094,0.06544738138766777,0.006211975083603987,8.76183301058252e-05,3.6862657482108066e-06,0.01267625945416885,0.005418806724771711,0.013704176635244118,0.00039420917812312814,,,,0.0019033486494012808,0.0007044513295972492,0.8873554552512541,0.006927552282255985,6.954328930705629e-05,1.9169546336514714e-06,0.003485845153332021,0.003333313810793645,0.012756444501676529,0.0001971283721271205,0.44333333333333336,1.7889808641066614e-17,0.21667774086378738,,,,,,,,,,6.734929816796876e-27,0.0,6.734929816796876e-27,Baseline
+249,0,KL Min.,Llama-2-7B,0.0,10,2,1e-05,0.7210884353741497,0.0,0.0,0.2875923780255352,0.03864225077602472,0.22840392152875602,0.027473468846892658,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.4740902104755777,0.03836163076543421,0.3665297446688551,0.030570073966038922,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.002776942628903413,0.000977362983639031,0.1379782740604028,0.01266606859205146,4.625652861676623e-17,3.7230710863853035e-17,0.00018074200217366542,5.945931546389664e-05,0.006263598561709471,0.00011462215606194333,,,,0.0017985867446393758,0.0006771314754316643,0.6966231723053505,0.012175197411290807,2.376017642941821e-17,1.3289697435231252e-17,0.00017282752484634064,6.051558988995322e-05,0.012001646829836817,0.0003745878400720144,0.27,2.2855015436448923e-10,0.2888150609080842,,,,,,,,,,5.01791941596116e-10,0.0,5.01791941596116e-10,Baseline
+250,0,KL Min.,Llama-2-7B,0.0,10,3,1e-05,0.7210884353741497,0.0,0.0,0.34106640240763725,0.0412382762971965,0.26913610432718743,0.032963919757285645,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.2956334447581319,0.031067389793786468,0.29074894802042156,0.020759035076338427,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.002776942628903413,0.000977362983639031,0.20305608991187224,0.016932144951760125,1.365815153894008e-23,7.554680429632255e-24,0.00018074200217366542,5.945931546389664e-05,0.006728421310414885,0.00016309396929761914,,,,0.0017985867446393758,0.0006771314754316643,0.7494544621813646,0.014073975311252462,1.1258589212096723e-29,1.1202829815281729e-29,0.00017282752484634064,6.051558988995322e-05,0.009667772980172738,0.0003365933686375298,0.35,1.6687859049967576e-12,0.26487264673311184,,,,,,,,,,1.0619204030347365e-16,0.0,1.0619204030347365e-16,Baseline
+251,0,KL Min.,Llama-2-7B,0.0,10,4,1e-05,0.7210884353741497,0.0,0.0,0.4349906763685168,0.041399677197376625,0.2617152784357442,0.025520569860729942,0.0,0.0,0.0064127027378722565,0.0002339181952486158,0.0,0.0,0.36218937769584114,0.03760897932258034,0.29083467610785985,0.025978663260389213,0.0,0.0,0.006258826904751285,0.00021636495911737754,0.0026436092955700796,0.0009694422922757615,0.16529649909914673,0.014232403245655625,2.0717691189905218e-08,4.243911736311607e-09,0.0001605399819716452,5.611579487497281e-05,0.015128848568515704,0.00028565751096415286,,,,0.0017985867446393758,0.0006771314754316643,0.6458844539685303,0.015464652835565633,1.574877261188353e-14,1.5596305250912948e-14,0.00017282752484634064,6.051558988995322e-05,0.009667772980172738,0.0003365933686375298,0.25,0.00019096774001327207,0.37574750830564785,,,,,,,,,,1.2161555648200794e-08,0.0,1.2161555648200794e-08,Baseline
+227,0,Pref. Opt.,Llama-2-7B,0.0,10,0,1e-05,0.8208616780045351,0.349,0.04658351177212351,0.5885007803449822,0.022577712211044373,0.4546292566307752,0.01255699842330583,0.0830920550038197,0.0118839011751025,0.9919856459330144,0.0030199106704405865,0.7905982905982906,0.03608780030180388,0.5409891780052368,0.030264113017313667,0.44226483755413365,0.018008410760658337,0.11507276198439438,0.008380926467572483,0.9506182440443186,0.0062716661445706935,0.06981717780134428,0.01275426151668499,0.4186071896064514,0.012160514176653668,0.820828101619634,0.004678683675417054,0.10807352309526223,0.015638901063356036,0.9955341619725498,0.0013901013983054322,,,,0.026014018621585054,0.007623378378633632,0.5910928697584164,0.011800412721158515,0.7964434666506258,0.00486268062395443,0.04845238095238095,0.009380518950538016,0.9988422337065748,0.0006519473829531803,0.3333333333333333,9.635514732762048e-15,0.24208194905869324,,,,,,,,,,3.709652809739326e-15,0.2993130236159747,3.709652809739326e-15,Baseline
+228,0,Pref. Opt.,Llama-2-7B,0.0,10,1,1e-05,0.8208616780045351,0.06199999999999999,0.02244844448804103,0.5588210977107396,0.026504838209564417,0.43523624742701317,0.014965691060851188,0.015976190476190477,0.005593054745790017,0.9550454545454545,0.004578952934133342,0.5683760683760684,0.04502066445053389,0.5171882622076496,0.032152150021254766,0.4174215613161179,0.019627539086695137,0.07427948779250088,0.007357732864079222,0.9303106045050152,0.006551093439922881,0.5267046457647624,0.024252315341516235,0.4230950475153902,0.013692146569582135,0.8755592645553156,0.006378099711610758,0.5259870440049652,0.02354571402058457,0.9305328277239128,0.003774976282447627,,,,0.061277851058873074,0.011322138338496426,0.5758466442889885,0.012288658847664221,0.8217270434008185,0.0062655927416635565,0.0954214724397645,0.012061225527843649,0.9831560905491679,0.001957671930141306,0.3,4.738095282671061e-17,0.22046511627906976,,,,,,,,,,2.5881511422773775e-12,0.2838313927741802,2.5881511422773775e-12,Baseline
+229,0,Pref. Opt.,Llama-2-7B,0.0,10,2,1e-05,0.8208616780045351,0.35733333333333334,0.04759084879353266,0.5526503168094474,0.028984945329307503,0.432423089003636,0.016968126698097875,0.07732421087684246,0.010949492158650851,0.9914490937488649,0.0028600194928174567,0.8219373219373219,0.03425908309815274,0.5120588272372638,0.03413604767313588,0.4143660071117207,0.021557008904321058,0.10318426428779445,0.006429412890896463,0.9293544474737612,0.006926052866525464,0.6918876122063032,0.021877922663137286,0.4506596788237609,0.014061372363312993,0.922050824252585,0.004567215577115462,0.7202470611987749,0.020433681366050577,0.9367089208028301,0.003780576309351217,,,,0.10374269568676657,0.012625188678157219,0.5454546541892364,0.012424495630398547,0.8456309344937554,0.005781223007970579,0.16647581665516448,0.011438035071703836,0.9922395302030801,0.001563280559848138,0.3333333333333333,7.009678794215372e-27,0.14248062015503876,,,,,,,,,,3.709652809739326e-15,0.5216721950958199,3.709652809739326e-15,Baseline
+230,0,Pref. Opt.,Llama-2-7B,0.0,10,3,1e-05,0.8208616780045351,0.6146666666666666,0.04803272360043588,0.56132185750228,0.029788705099224072,0.4395546536808594,0.01780238662649257,0.13263268018531177,0.011022724913657422,0.9573988051131264,0.004703249107181397,0.8133903133903134,0.03493560388692606,0.5194528571782538,0.03454344919906302,0.4202118466684467,0.022004658342985147,0.10268375110813233,0.0065937428964228445,0.9200466536369167,0.006416712917155158,0.7625219857942667,0.01961085128404821,0.46100548756873716,0.01412034058667831,0.9357573116100575,0.003974188778366592,0.7824214660978699,0.01899557984575811,0.9259736441010563,0.003300053055703975,,,,0.06082510876354154,0.01144722200523549,0.5398006093087906,0.012625245614030311,0.8276419618300567,0.005944180966896035,0.08878749953749954,0.011682944694107085,0.911264961040553,0.0009917005596740411,0.33666666666666667,2.1434549009040572e-30,0.11838316722037652,,,,,,,,,,1.8502009026131303e-15,0.5708133237545386,1.8502009026131303e-15,Baseline
+231,0,Pref. Opt.,Llama-2-7B,0.0,10,4,1e-05,0.8208616780045351,0.7313333333333333,0.04383885397504271,0.548320668433842,0.030488207658856525,0.43134591555313534,0.01846937345275885,0.148571563369551,0.010099760158172844,0.9707312395134835,0.004437173753020781,0.8561253561253561,0.031206459854159443,0.5118411253605883,0.03497879989530106,0.4164981001571263,0.022367768113513594,0.10838170717274538,0.00650043780546534,0.9219647745882619,0.006723696287322877,0.8138883783454983,0.01733657859425645,0.46183777595680436,0.014262868141484798,0.9446237675099981,0.003922565985447677,0.8318081450030967,0.01643946160397134,0.9270252494915902,0.003513479566589719,,,,0.04825888777440945,0.010231256776888057,0.5416982368540082,0.01284763333924324,0.8138999759523842,0.006169204440312425,0.08293497632971317,0.010706712036329922,0.9661635410942686,0.00260308978036796,0.3233333333333333,8.097382402130872e-30,0.12223698781838317,,,,,,,,,,2.858944731506222e-14,0.5812169529105777,2.858944731506222e-14,Baseline
+253,0,Retain Model,Llama-2-7B,0.0,10,-1,1e-05,0.0,0.9279999999999999,0.024166091947189144,0.5947988854000636,0.033605223087774444,0.44980359540221637,0.02234923922112359,0.17411979111359915,0.006795951960130148,0.9719513071314938,0.005006360676130846,0.9088319088319089,0.025711199561382098,0.5459951074987635,0.03578297274991433,0.42124277033515506,0.02677498414458228,0.1031676073808966,0.005784872630771222,0.9140786084176485,0.0071238723761199345,0.9810537904728946,0.005545055365200779,0.47393877357373354,0.014249411674370585,0.9893784260662525,0.0007522381003083327,0.9838158979130903,0.004790903384963573,0.9246379981408999,0.0032924872531743506,,,,0.39422382472862666,0.010360220997732228,0.6711315665909536,0.012556145190908048,0.1460792032498096,0.007758181724581559,0.3957377806201532,0.010624726080647672,0.904281838859758,0.004018981875738075,0.0,1.0,0.0,,,,,,,,,,1.0,0.6237955595741369,1.0,Baseline
+232,0,Grad. Ascent,Llama-2-7B,0.01,10,0,1e-05,0.8888888888888888,0.945,0.02105188395801711,0.6187487585193752,0.029973667964308982,0.47501447977915695,0.02007173148500605,0.18622038513601982,0.006581976367816515,0.9708367168970651,0.005188968582688854,0.8917378917378919,0.02724332888874447,0.595287258859532,0.034329055471394955,0.4748396945412093,0.02928818019336817,0.10978678007162364,0.006239153050772274,0.924176642795394,0.007383537167726773,0.7076934448033372,0.01578149214116453,0.4573690568506355,0.012538971655043264,0.871716620502705,0.004754612902815609,0.7192132372798172,0.016004285369496184,0.9058151344389838,0.004756613728251303,,,,0.5984424176148694,0.015336641613849764,0.55819401989677,0.012319882647525952,0.7875850838858941,0.008101292071784954,0.6259319067792977,0.015904413181527088,0.9016045590506145,0.00475421784960993,0.3466666666666667,1.0956261248491814e-16,0.22376522702104099,,,,,,,,,,2.1942743021891237e-16,0.6235525384005506,2.1942743021891237e-16,Baseline
+233,0,Grad. Ascent,Llama-2-7B,0.01,10,1,1e-05,0.8888888888888888,0.0,0.0,0.19017189691853528,0.025749612454904293,0.26578291541651766,0.014908871829083703,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.16566073502922174,0.020870095070104136,0.2681023967225485,0.013047213862247763,0.0,0.0,0.011758684425253466,0.0001587906083539704,0.0,0.0,0.03235800970366027,0.0027224033868851636,9.366409784245131e-05,1.1151789076700771e-06,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9459451648127121,0.002712272355774447,9.271086699894434e-05,1.2201016457170914e-06,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5333333333333333,1.4973669324808988e-17,0.21599114064230343,,,,,,,,,,2.0885153422421613e-39,0.0,2.0885153422421613e-39,Baseline
+234,0,Grad. Ascent,Llama-2-7B,0.01,10,2,1e-05,0.888888888888889,0.0,0.0,0.13636626883647138,0.018218438084703144,0.2548431626364454,0.009154751896633815,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.1314631652917845,0.018396323316543572,0.2551124401152376,0.009872690261049196,0.0,0.0,0.014051087744462232,0.0007150453334525698,0.0,0.0,0.025344187064717336,0.002152979317936579,6.263111280117667e-05,5.858673320057006e-07,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9501198892873849,0.0027787470753550198,5.5653963975915514e-05,9.966222187258457e-07,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5533333333333333,8.607191677387933e-18,0.21386489479512735,,,,,,,,,,1.4228925129969089e-42,0.0,1.4228925129969089e-42,Baseline
+235,0,Grad. Ascent,Llama-2-7B,0.01,10,3,1e-05,0.8888888888888888,0.0,0.0,0.5552698981078624,0.04236402576093415,0.42725355015714167,0.04177364286477556,0.0,0.0,0.035205880111707506,0.0005093501879041153,0.0,0.0,0.5218883184769632,0.03823529418057141,0.43497967266153775,0.03877145287550543,0.0,0.0,0.028361574978966436,0.0005039275343647508,0.0,0.0,0.24654598165895852,0.019055973974939425,5.863093817103905e-23,2.4758018480035482e-23,0.0,0.0,0.03826366293740615,0.0002439897706441564,,,,0.0,0.0,0.43957046500086866,0.017909319836362247,6.459840309107607e-18,3.498821365726045e-18,0.0,0.0,0.038111203951432915,0.0002974244900519239,0.31333333333333335,0.03300651266258098,0.4289922480620155,,,,,,,,,,2.0608068972918464e-13,0.0,2.0608068972918464e-13,Baseline
+236,0,Grad. Ascent,Llama-2-7B,0.01,10,4,1e-05,0.8888888888888888,0.0,0.0,0.504849139257392,0.04527589534556345,0.31376372681284725,0.039914232727733125,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.48067544880832946,0.04225244571720282,0.3726199292098841,0.04128048893549813,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.1942667381581949,0.016367476156085616,1.3060374368858345e-50,9.625248255663002e-51,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.5107680628113074,0.016206167692476032,9.991660241684096e-51,7.823474588255901e-51,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.26666666666666666,0.058676762413160534,0.4370321151716501,,,,,,,,,,8.690124079460009e-10,0.0,8.690124079460009e-10,Baseline
+212,0,Grad. Diff.,Llama-2-7B,0.01,10,0,1e-05,0.8208616780045351,0.9329999999999999,0.02378587136868601,0.6044341571342846,0.03295403844927024,0.46839407555481827,0.022090022496097518,0.1796095700824803,0.00715842576061889,0.9731410232901755,0.005123464894959443,0.8703703703703705,0.03001669034410197,0.5754631800007903,0.03523088213461695,0.452703619302839,0.03157332063030803,0.10250601652761857,0.0063887216818942945,0.9168854619122493,0.0068838846483578854,0.9240141808908553,0.010863017541088014,0.47780056612410154,0.013769749526114948,0.9645564544738333,0.0034398398192402677,0.9239827454191956,0.011024987548198488,0.9211929721771196,0.003438739275478625,,,,0.7628701916181125,0.016218603493237366,0.5214295985678726,0.012881445771069767,0.8830187610028744,0.007961448278010633,0.7761175244208777,0.01570711610614146,0.9164478433442056,0.003852978249210618,0.38,6.200544599492621e-21,0.18737541528239202,,,,,,,,,,1.096624314778916e-19,0.6352017316602937,1.096624314778916e-19,Baseline
+213,0,Grad. Diff.,Llama-2-7B,0.01,10,1,1e-05,0.8208616780045351,0.5578333333333333,0.04730234142120421,0.597399076284999,0.03657226137799652,0.4372842183430803,0.026876328726907105,0.1062610336840886,0.01016555309682237,0.9210354922631803,0.014897400813727698,0.8347578347578347,0.03233952540308097,0.6076858141004515,0.03700330996462044,0.4548952485727426,0.037325885029387225,0.1081939624630904,0.006231085124890657,0.9330963922463212,0.007325057987164338,0.2820281513217174,0.010302055954323147,0.4720819075840925,0.017541910057826682,0.12861225661177608,0.010266014417502141,0.3130985505562747,0.01125824855355396,0.8151398983349183,0.011366509599946563,,,,0.22890351582075502,0.008735109251248518,0.5166760164328829,0.015120346406399494,0.02797749589839668,0.004167421910138012,0.2573758111316908,0.009842303873877361,0.6918125042115183,0.01652302094735894,0.20666666666666667,2.9465612052258827e-05,0.3608637873754153,,,,,,,,,,5.0749555303276145e-06,0.37136995954344854,5.0749555303276145e-06,Baseline
+214,0,Grad. Diff.,Llama-2-7B,0.01,10,2,1e-05,0.8208616780045351,0.147,0.03327636546133754,0.7152883532832506,0.03462380372836556,0.5848631109936556,0.03369161222468478,0.01565450698611169,0.005738428795905251,0.11994744747643918,0.020166710875855913,0.5893162393162393,0.04339049368207918,0.6896381931427219,0.03443033329209034,0.5141956225115176,0.03477084707033165,0.04905851146179271,0.005739531894616003,0.42710824073297227,0.03218318019420482,0.07379010824554239,0.004775792187145151,0.4530032901235258,0.016115559800401306,0.01937100024945274,0.0030592455262335766,0.03778894668639581,0.0035858207485331348,0.1881169109791714,0.0098784532822644,,,,0.0026808296270702282,0.0009656265843762572,0.6751735614722527,0.015665786459479542,3.030613392968039e-08,2.46788550568028e-08,0.00042739613704646703,0.00013117332020543297,0.006863812247074542,0.00017425482045506463,0.24,7.617974544503307e-05,0.3682392026578073,,,,,,,,,,5.439158189288132e-08,0.10924283628908843,5.439158189288132e-08,Baseline
+215,0,Grad. Diff.,Llama-2-7B,0.01,10,3,1e-05,0.8208616780045351,0.4771666666666667,0.04749898398110019,0.7279976089525759,0.03485902716106826,0.5954786744821959,0.027806472862790126,0.07511874282242166,0.009413410996968179,0.5351868789269048,0.03108794704562312,0.809116809116809,0.0350001378456764,0.6468973396585224,0.03524502096591484,0.4999016951021712,0.03475473510530182,0.05772005979528125,0.004674357775533508,0.6511698504635343,0.02360279378091694,0.3096741126054053,0.010965745275366099,0.4591469667354204,0.015298215551960445,0.26800962161977326,0.010061193205802456,0.22817042640332877,0.007961236148395,0.6918101315053312,0.012769405424800916,,,,0.0021308296270702285,0.0007583302993385423,0.6305831982904452,0.016256156952678267,1.5056979096577731e-09,1.5042876151617233e-09,0.0003659630516579701,0.00011402112121218639,0.006693008174996586,0.00010315359489165173,0.2633333333333333,0.00011353306693858288,0.37145071982281286,,,,,,,,,,1.4942294797261553e-09,0.47196606929168716,1.4942294797261553e-09,Baseline
+216,0,Grad. Diff.,Llama-2-7B,0.01,10,4,1e-05,0.8208616780045351,0.469,0.04751160293043248,0.7453841996446183,0.03327244376567869,0.5984568053310647,0.028215022482907977,0.07322183830081555,0.009504423766840535,0.4764790847453708,0.03451644085805309,0.8148148148148149,0.034321831060000335,0.6384253136147289,0.03511364017528178,0.4942245042890296,0.03471004999289998,0.059527793303241545,0.005028303362985733,0.6364918224421716,0.02581335354702919,0.35577236819922264,0.01162188868264195,0.46303656688553985,0.015838594293178326,0.38230518866783353,0.011643398799723774,0.2717274228601494,0.00978831627147335,0.736477445898485,0.013017131500073496,,,,0.0009965703678109694,0.0004834320048865688,0.6212948598570626,0.01634484834968054,6.127305690859893e-17,6.127305690859798e-17,0.0002410065944908113,8.079507483856646e-05,0.006396075534510508,4.5224150597273095e-05,0.24,0.0002507290106945618,0.37805094130675526,,,,,,,,,,5.439158189288132e-08,0.5128817919151487,5.439158189288132e-08,Baseline
+242,0,KL Min.,Llama-2-7B,0.01,10,0,1e-05,0.7210884353741497,0.935,0.023066657908563763,0.6133718552480142,0.030304737158523016,0.4707526603209077,0.02034364972398901,0.18233871125434592,0.006957184240937999,0.9715017415276072,0.005089830573148239,0.8917378917378919,0.02724332888874447,0.5892156089778035,0.034574670045988284,0.4711302936223364,0.029618872617726534,0.10932319695344021,0.006239541420652388,0.923197957862593,0.0073699327347598875,0.7404904100854898,0.015514693358977427,0.4612156167410793,0.012596562072535716,0.8846310673739973,0.004609736816808154,0.7449481316280455,0.01575808557430623,0.9082767535040845,0.004804146250715584,,,,0.6133219133758041,0.015676852894546976,0.554160597170425,0.012345522594458872,0.7955417828268352,0.008328697839953632,0.6394357786029153,0.016100041853064496,0.9030354865198414,0.004630880510777976,0.3466666666666667,2.81005469471046e-17,0.21842746400885935,,,,,,,,,,2.1942743021891237e-16,0.624380140246935,2.1942743021891237e-16,Baseline
+243,0,KL Min.,Llama-2-7B,0.01,10,1,1e-05,0.7210884353741497,0.0,0.0,0.238464139683634,0.029646684960232,0.2947606897739976,0.01761371602350222,0.0,0.0,0.011808436522583825,1.5631146296399426e-05,0.0,0.0,0.18568873407805564,0.022450669084094,0.2798750675153093,0.014223067967476515,0.0,0.0,0.011561045688060979,3.304553176102566e-05,0.0,0.0,0.03213729194016307,0.0027016841484961023,9.35232698524954e-05,1.04836771141367e-06,0.0,0.0,0.011914887315111315,1.632152390870383e-05,,,,0.0,0.0,0.9464136463909026,0.0027191624904088275,9.336259214701391e-05,1.2385687783025482e-06,0.0,0.0,0.012628864590307532,3.7973767262187355e-05,0.5366666666666666,1.7685834282543408e-17,0.21663344407530455,,,,,,,,,,6.3425628614729406e-40,0.0,6.3425628614729406e-40,Baseline
+244,0,KL Min.,Llama-2-7B,0.01,10,2,1e-05,0.7210884353741497,0.0,0.0,0.3936456744555,0.036225724289654475,0.350643514377555,0.025719581499431092,0.0,0.0,0.0530176844286043,0.00206604726826176,0.008547008547008548,0.006017541110844508,0.39845067794737415,0.03562419862189332,0.39062422906428024,0.030081709237959546,0.00013108292577090322,9.232299997272141e-05,0.0534071180420904,0.001991000156626323,0.00669847746329261,0.0015099302861793134,0.03193739158008537,0.0025645952566283733,8.76876952298274e-05,3.450073150153303e-06,0.005359026898977892,0.0018216827374817866,0.05866748730238602,0.0015286622464291207,,,,0.0003935185185185185,0.000278277323433988,0.943652872803225,0.0031943976121007267,4.5601102193978194e-05,1.2222629383366072e-06,0.0004166666666666667,0.0003433216436840604,0.02409383462708579,0.001189833762448708,0.5466666666666666,2.7624167927647097e-17,0.2183610188261351,,,,,,,,,,1.6793386636236394e-41,0.0,1.6793386636236394e-41,Baseline
+245,0,KL Min.,Llama-2-7B,0.01,10,3,1e-05,0.7210884353741497,0.0,0.0,0.5321772302857819,0.04628307817096065,0.37136380861364576,0.04249495012316642,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.49960339782644414,0.04259371941976864,0.3528943720722415,0.041515594572534696,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.19925104640484928,0.016512874204130022,7.271822890267727e-42,6.008798109027888e-42,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.4848340319767787,0.01662580386285543,1.4490835507200026e-43,6.326111196707916e-44,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.2833333333333333,0.07374426336252336,0.44044296788482834,,,,,,,,,,5.191390347437448e-11,0.0,5.191390347437448e-11,Baseline
+246,0,KL Min.,Llama-2-7B,0.01,10,4,1e-05,0.7210884353741497,0.0,0.0,0.49764014345177116,0.046819008985518336,0.2843347363522162,0.0389223340713372,0.0,0.0,0.01771265478387573,2.3446719444599106e-05,0.0,0.0,0.48041996668378434,0.043065061323862515,0.35049865513981787,0.04060085348588588,0.0,0.0,0.01734156853209147,4.95682976415385e-05,0.0,0.0,0.17853184237378694,0.01569468977387372,4.715506085151057e-61,3.4800339636551286e-61,0.0,0.0,0.017872330972666977,2.448228586305574e-05,,,,0.0,0.0,0.5101658771973803,0.016075024386026513,4.135968240130737e-62,1.950218244233312e-62,0.0,0.0,0.0189432968854613,5.696065089328104e-05,0.27,0.04165921875451196,0.4321594684385382,,,,,,,,,,5.01791941596116e-10,0.0,5.01791941596116e-10,Baseline
+222,0,Pref. Opt.,Llama-2-7B,0.01,10,0,1e-05,0.8208616780045351,0.37233333333333335,0.047045253663423825,0.5747826704930294,0.022849857100624225,0.44880716453005887,0.012946559289478082,0.0868266814884462,0.011641537628901816,0.9906761221234907,0.003133851567632787,0.7905982905982906,0.03608780030180388,0.5303338377740737,0.03049196810163221,0.441235403554138,0.020838890900462138,0.11602329651697596,0.008335675165517583,0.9533925007479708,0.005961710553429642,0.07434148604521824,0.013029736462718338,0.42049895460469316,0.012108259682437481,0.8160908256066413,0.0046802303284757705,0.1080846924814316,0.015462345403289187,0.9936868898939095,0.0017010734139741486,,,,0.028277057994119316,0.007965292118686412,0.5930864113944144,0.011841383325235945,0.7879623341996244,0.004929242570194882,0.05130952380952381,0.00989755530564852,0.9990878477416626,0.0005683075029403473,0.3233333333333333,1.1492587912901443e-10,0.2853156146179402,,,,,,,,,,2.858944731506222e-14,0.30900825088215067,2.858944731506222e-14,Baseline
+223,0,Pref. Opt.,Llama-2-7B,0.01,10,1,1e-05,0.8208616780045351,0.06199999999999999,0.02244844448804103,0.5503895341459227,0.027529462899201208,0.4349964050751066,0.015904772910049554,0.015654761904761904,0.0054748669113853345,0.9749580419580419,0.004196233231343522,0.5826210826210827,0.04481425552815165,0.49997377594307063,0.033465782078326387,0.4223165442478161,0.023514937537052354,0.07762327979059412,0.007440879266492267,0.937517938121109,0.006353850813122646,0.4824519774260634,0.025888887482011673,0.423752270204518,0.01371076968041236,0.8777211397151145,0.0062684662493249985,0.4844270433615668,0.024849760334561682,0.9465048078425086,0.0038751731913320474,,,,0.04980129955723792,0.009644963435539073,0.577464211841622,0.012441048291071273,0.8196756259521027,0.005915193258850345,0.0814799254101586,0.009659806188445728,0.9923094672768938,0.001879361634375717,0.29333333333333333,3.8567598361995186e-11,0.27986710963455147,,,,,,,,,,8.777796874606567e-12,0.28212189963542056,8.777796874606567e-12,Baseline
+224,0,Pref. Opt.,Llama-2-7B,0.01,10,2,1e-05,0.8208616780045351,0.29133333333333333,0.04497316820711943,0.5475407125684502,0.028960406822220543,0.43162688013468,0.017564841653853462,0.06354737660000818,0.010517512451336818,0.9890253719587336,0.0032539991601581884,0.7934472934472935,0.036622385080893015,0.49517917025551594,0.03476778421692807,0.4190363239807755,0.025108561239523465,0.10279450910247236,0.006699578536422261,0.9411200021821509,0.006183728525904985,0.6828504177730427,0.022262394811320772,0.446238155102904,0.014013804475529752,0.915881406542059,0.004959978994002249,0.7057888619841924,0.021200239034826405,0.9356739696374873,0.003783871748562699,,,,0.10198269458313836,0.013198848712539986,0.5537857026468429,0.012784410679399106,0.8439264730253759,0.00542037471883288,0.16414206237232554,0.01258984275626304,0.9925432123462611,0.001605157679267544,0.32,1.9053339910013803e-14,0.24498338870431893,,,,,,,,,,5.563730678470443e-14,0.498915403815241,5.563730678470443e-14,Baseline
+225,0,Pref. Opt.,Llama-2-7B,0.01,10,3,1e-05,0.8208616780045351,0.5846666666666667,0.04865744112548367,0.5400212376424239,0.029669493442535416,0.42687285242145395,0.018235922311307263,0.12580395335658495,0.011394220612185809,0.9523487188344092,0.00482466802738278,0.8390313390313391,0.03280675186486188,0.494666242119652,0.035019207228537734,0.41800922410335023,0.025548387823867254,0.10402994944230111,0.00629836411628782,0.919879552552894,0.0065607985781069465,0.774547418438988,0.02032252084544217,0.4551786813511306,0.013984467524247847,0.9368884339109415,0.00380461750460525,0.7862055863021258,0.019551089894799253,0.9258696608260304,0.0033838746869705393,,,,0.08397916876683457,0.013218894282992845,0.5477669687086352,0.012911001249366085,0.839265697788524,0.005362487991475072,0.12287430065061644,0.013592981224766003,0.9322256101284517,0.0023143343324680196,0.33666666666666667,1.4785541822365532e-15,0.23426356589147287,,,,,,,,,,1.8502009026131303e-15,0.5601941167018999,1.8502009026131303e-15,Baseline
+226,0,Pref. Opt.,Llama-2-7B,0.01,10,4,1e-05,0.8208616780045351,0.6496666666666667,0.046780752802537996,0.5358880903702782,0.029964606084025514,0.4246043102292606,0.01854252964929691,0.13681449836712994,0.010917621513464672,0.9595042212267535,0.004761058803675313,0.8475783475783476,0.03202643916106147,0.49298545701035773,0.03514675499374474,0.4169656011526384,0.02575251703752355,0.1034027130556435,0.006253261375128168,0.9196098342140809,0.006836431821863783,0.8235664994975654,0.017909103149189986,0.4573389752774102,0.013994122673080957,0.9450982542002703,0.003475079029212093,0.8384847297951948,0.017008985636463032,0.926449565804033,0.0033328728586064175,,,,0.06266997094265174,0.011774912931646194,0.5469193067493091,0.012932046786020229,0.8366279850677959,0.005404314441473742,0.08752151211361738,0.011839264225030333,0.911084098266416,0.0009251037039683804,0.34,8.6557217874757e-16,0.23207087486157255,,,,,,,,,,9.158954716915745e-16,0.5686880809848377,9.158954716915745e-16,Baseline
+252,0,Retain Model,Llama-2-7B,0.01,10,-1,1e-05,0.0,0.9179999999999999,0.02587381969200809,0.5867992311686108,0.034162794309235134,0.4445423697965439,0.022638593539283134,0.1737265650586084,0.007311711447675104,0.9724468340248467,0.005002158226125613,0.9088319088319089,0.025711199561382098,0.5468444548453482,0.03562985978961973,0.41868767541998175,0.026761345244793615,0.10158596746207914,0.0057138691806951055,0.9139116515846981,0.007144110342348025,0.9822053126894559,0.005094663869762722,0.471867662555522,0.01432314572853207,0.9897651615521136,0.000750840077271583,0.9846385499993602,0.004445153516169368,0.9240800326247677,0.0033220938143150103,,,,0.3983512708804085,0.010367448527135203,0.6694346204288318,0.012469078839573993,0.14361598711400606,0.007708751753109203,0.4025678758675452,0.010442822696508139,0.906735889275128,0.003856405682599201,0.0,1.0,0.0,,,,,,,,,,1.0,0.6203436494291608,1.0,Baseline

versions/llama-1p/llama-1p.csv

@@ -0,0 +1,44 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+254,0,Finetune Model,Llama-2-7B,0.0,0,-1,1e-05,0.0,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9844665453461495,0.0051727714413351816,0.5021345577886491,0.013075097432894608,0.9905248646281772,0.0007461848129786882,0.9876392022245997,0.004309388867036841,0.9158566386007846,0.0035939738499808516,0.38,4.709184258054772e-44,0.036345514950166115,,,,,,,,,,1.096624314778916e-19,0.6285644571327744,1.096624314778916e-19,Baseline
+153,0,Grad. Ascent,Llama-2-7B,0.0,1,0,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
+154,0,Grad. Ascent,Llama-2-7B,0.0,1,1,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5900429345628848,0.033905759300725195,0.4547378833757618,0.022335305669671995,0.1755233138034995,0.007054415039534881,0.9705581657460249,0.005055939888176935,0.8874643874643875,0.02809493708284436,0.5601436387215902,0.03594055775086999,0.4311578250642015,0.026881223608336224,0.101544103709312,0.006030005405901275,0.9125813125165163,0.0071185045685729365,0.9889397136536354,0.0037059751472614064,0.48344775524839767,0.01405782374996479,0.9872371886220882,0.0014907038206082325,0.987697022848052,0.004142802637345347,0.9232821917154683,0.0034525307216189344,,,,0.7761951905387826,0.03772874843774116,0.5348676866651111,0.034412833437561345,0.8994496957903285,0.01639860932264404,0.7697821692084698,0.03782303934764471,0.9036587801876312,0.009903910085434673,0.425,3.489894879749045e-07,0.08048780487804878,,,,,,,,,,0.0012708143485281624,0.6298261119892148,0.0012708143485281624,Baseline
+155,0,Grad. Ascent,Llama-2-7B,0.0,1,2,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5873988265455783,0.03397508491678948,0.4508092646908872,0.022551229274736694,0.1739081567383425,0.007062546425329464,0.9665255339610767,0.005908090892117214,0.8760683760683761,0.029119795619916317,0.5583610983349427,0.03599265069970819,0.4281143594726287,0.026936659748423613,0.09949787624348999,0.005927416138043877,0.9107472606370924,0.0070831369098684925,0.9829393276544355,0.004747087133082975,0.4874752426550318,0.014033035101973904,0.9855465733622553,0.0018392288196213203,0.9823404390280708,0.0048268090647223105,0.9234222857023673,0.00337002024051439,,,,0.6634498511248423,0.03979501310983266,0.5330159294477221,0.03463020113629034,0.7603332595788647,0.033966656011972296,0.6583112962777088,0.039468344041597575,0.9078713971845589,0.009329761694848934,0.425,2.0396328181959689e-07,0.07439024390243902,,,,,,,,,,0.0012708143485281624,0.6274349162486959,0.0012708143485281624,Baseline
+156,0,Grad. Ascent,Llama-2-7B,0.0,1,3,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5767494734917964,0.034372249724000214,0.4408938678030151,0.02287028723794112,0.1750376753678611,0.007158422081483091,0.9663828541959902,0.00575613164904337,0.8917378917378919,0.02791127949453827,0.5513137330681693,0.03617204964693906,0.42021024790475425,0.027096090976780653,0.09769306113650794,0.0059427576527659514,0.9055722933002827,0.007772959154843439,0.9616573619305047,0.006924760552611335,0.4892999713770692,0.014168689417265349,0.9774506636931869,0.0028459379602567628,0.9583282920447302,0.007198221797581762,0.9220386919418376,0.003423647927837329,,,,0.5121226937822042,0.032962969948126364,0.54345131884367,0.034968297994459455,0.43023176455719464,0.04345304441763552,0.470313608214277,0.03389656752927492,0.8711500171294493,0.010297842300251473,0.375,5.841629899805412e-07,0.08658536585365853,,,,,,,,,,0.006760732303569208,0.6208832381499613,0.006760732303569208,Baseline
+157,0,Grad. Ascent,Llama-2-7B,0.0,1,4,1e-05,0.98989898989899,0.943,0.02184680361707013,0.569912690980619,0.03466840517291862,0.43204434512382306,0.023307965482841985,0.17792390999093788,0.007721143710477113,0.9666332582272671,0.0057618895495265676,0.8831908831908832,0.028916035930034605,0.5473668144428057,0.03622314683593883,0.414897165484955,0.027344081869356943,0.09466068410867952,0.006050371078030876,0.8990135186717393,0.007874996621043741,0.9360858953948274,0.009325230518351905,0.49302946184701624,0.014332776372096316,0.9674047600471735,0.003620235746904036,0.9232295288474817,0.01000286376090822,0.9195202169991762,0.0035090468594729846,,,,0.4726138841044719,0.03209104244253,0.5596537628394563,0.038582904584907325,0.273127242585076,0.040910247561870554,0.4193707732983415,0.03496432463333179,0.8648833328747095,0.010606785895688082,0.35,2.8431650207494386e-05,0.14146341463414633,,,,,,,,,,0.01430154804770646,0.6152017367360003,0.01430154804770646,Baseline
+133,0,Grad. Diff.,Llama-2-7B,0.0,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
+134,0,Grad. Diff.,Llama-2-7B,0.0,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5831342368229205,0.034195526749926465,0.4463399542137994,0.022676754446149583,0.1741332194637148,0.007005056236897536,0.9711427967298144,0.004892496292632562,0.8746438746438747,0.02986595450792445,0.5553851084435623,0.03611651818945058,0.42606843965321073,0.02705610226448954,0.09996325669567287,0.006159195821413578,0.9108871819027446,0.007062997168334457,0.9881749522878556,0.00403912406410768,0.48451691239938044,0.014172133539883811,0.9886402516003234,0.0015158301664681946,0.984391926889841,0.00497080235933812,0.9225775765926555,0.0033693716971956124,,,,0.8470500248532187,0.03425612789078735,0.5321108595821069,0.0345352995124133,0.9437253641548825,0.010302771440246095,0.8551610918357241,0.0338847158578802,0.9129679236930579,0.008755089796400987,0.425,3.489894879749045e-07,0.08048780487804878,,,,,,,,,,0.0012708143485281624,0.6247652337390697,0.0012708143485281624,Baseline
+135,0,Grad. Diff.,Llama-2-7B,0.0,1,2,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5743689863204028,0.03459778800824567,0.4395178398225532,0.02291569750507518,0.17573586844871675,0.0071367315805213145,0.971376699089106,0.004833659056828999,0.8846153846153846,0.02814097500969468,0.5500806928206896,0.03629220886455765,0.4206527134660717,0.027177369607437277,0.10089550395690719,0.005891426629145233,0.9103603531622805,0.007225732932586464,0.9876166503747148,0.003958159615863906,0.48619738136346835,0.014218352461945517,0.9870674423893376,0.0017276553352510255,0.9834584001935912,0.005012665963437899,0.9230005342696567,0.003320496577337283,,,,0.693891532134268,0.040761864873975456,0.5313708135093245,0.03468764401794746,0.7907704753851272,0.03193867027599963,0.697210810027939,0.0408282479347697,0.9117740693087659,0.008923458576449092,0.425,3.1401395972352475e-07,0.07926829268292683,,,,,,,,,,0.0012708143485281624,0.620856677526391,0.0012708143485281624,Baseline
+136,0,Grad. Diff.,Llama-2-7B,0.0,1,3,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5588856115278307,0.03486192436995179,0.42454388004631033,0.023382520023456987,0.17113257408226445,0.006696328375983813,0.971384233041181,0.004872506472042895,0.8660968660968662,0.030161453599377153,0.5440843064984168,0.03625778397852338,0.4103980888734902,0.02752113340668667,0.09743177268489672,0.006137860799523201,0.9075885179544705,0.007384150110357787,0.9567632211441702,0.008380284589874188,0.4919675035655331,0.014354171258223164,0.9788901898126867,0.002587745577928834,0.9498857465693834,0.009150898069415532,0.9209175410960485,0.0035039275462926496,,,,0.5611743395394859,0.0361031249078284,0.5357745264531568,0.035138959954993894,0.48186478504437824,0.04263428759390712,0.5391800516220583,0.0371989002115956,0.8954952252076869,0.010479046564108179,0.4,1.055741449818015e-06,0.09390243902439024,,,,,,,,,,0.003018184077228396,0.6098886454382855,0.003018184077228396,Baseline
+137,0,Grad. Diff.,Llama-2-7B,0.0,1,4,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5495203912493695,0.03510662588135505,0.41377046154740876,0.023744863866831466,0.16752673547642588,0.006608755494593228,0.9705274573313899,0.005449214872377944,0.8703703703703705,0.03001669034410197,0.5372712890340352,0.036215753077327884,0.40173797204393547,0.02764724919841151,0.09540026299698359,0.006152647895052731,0.9050756487748767,0.007502582140361348,0.916460247115961,0.010795592976781397,0.49330805534207384,0.014632422453902047,0.965061026568477,0.0034352591571064324,0.9011082675444243,0.011800933253074398,0.9188436974164093,0.0036434938209770657,,,,0.49253655875189073,0.03322129247825073,0.542568072727047,0.03635081642015713,0.2987510252217606,0.040704356395482096,0.442433429498021,0.03283629382771646,0.8643185765471209,0.012045353707243911,0.35,1.6715837773517706e-05,0.1329268292682927,,,,,,,,,,0.01430154804770646,0.601047475651797,0.01430154804770646,Baseline
+163,0,KL Min.,Llama-2-7B,0.0,1,0,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
+164,0,KL Min.,Llama-2-7B,0.0,1,1,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5926914370142016,0.03380708048334772,0.4552599427511621,0.022397353324240442,0.17548038451057024,0.007054889176896772,0.9701414990793583,0.005048595089360008,0.8917378917378919,0.02791127949453827,0.5606620010502634,0.03589288783261826,0.4313463368160679,0.026893905130676334,0.10109508486948855,0.006048669438358328,0.9115536136563003,0.007054328446766984,0.9866427948861284,0.004164280248771519,0.48380044848689596,0.014028587577548995,0.987079635387015,0.001524838575423045,0.9856550808912471,0.004541405936506324,0.9234270631972602,0.003406348491488958,,,,0.7786513675263843,0.038214845616106496,0.5342827939227608,0.03416607249348546,0.9022485278034719,0.01636718106068851,0.7863602232753086,0.03872457119116467,0.9052995628926588,0.009539580654677837,0.425,4.299272404750809e-07,0.08292682926829269,,,,,,,,,,0.0012708143485281624,0.6305829841958074,0.0012708143485281624,Baseline
+165,0,KL Min.,Llama-2-7B,0.0,1,2,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5852513278783095,0.03415700434871471,0.4512026997232316,0.022526850296365288,0.1744369057670915,0.007011106460219607,0.9684490075591609,0.005697520028061312,0.878917378917379,0.02908252649809801,0.5583273505371179,0.03607477677978727,0.42904895977528,0.027004029214984314,0.10027173126835717,0.006095704998244655,0.9102069325105575,0.0071130182494677984,0.9837020298645062,0.004679527798699983,0.4863827807838256,0.014036072547985811,0.9851094086008342,0.0019048629397133007,0.97964844544246,0.005470548235405333,0.9232841112517135,0.003354321867024587,,,,0.6452102127494272,0.038795734192378806,0.5339878688212196,0.03469282005007738,0.7561577607136813,0.033841205404102265,0.640837113817565,0.039597880960097746,0.9005021575058872,0.009514024034688795,0.4,1.634398358874023e-07,0.07195121951219512,,,,,,,,,,0.003018184077228396,0.6274392149443913,0.003018184077228396,Baseline
+166,0,KL Min.,Llama-2-7B,0.0,1,3,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5771976816818047,0.034452997218025624,0.4406256089564199,0.022935863913454847,0.17579360511063297,0.007156745309643807,0.9661798328164983,0.005836219904184736,0.878917378917379,0.02908252649809801,0.5505496096178601,0.036126357568608575,0.42045603527182135,0.02714191375993806,0.09428626024741103,0.006039913345791347,0.9011634386488793,0.007877591930412776,0.9631817083850834,0.006755508048376161,0.4901608167131136,0.014162208578683038,0.9758064280747213,0.0031446263998563873,0.9574773675645152,0.007325667723048805,0.9220544162502203,0.0034188156565595406,,,,0.5208700880045747,0.033876535953684686,0.5440980827111798,0.035182167822362186,0.42186229134424025,0.04294040838303995,0.4713160054717275,0.03472033829656913,0.8696356371868526,0.011975877633282655,0.4,1.6943467926466838e-06,0.1,,,,,,,,,,0.003018184077228396,0.6202838445189751,0.003018184077228396,Baseline
+167,0,KL Min.,Llama-2-7B,0.0,1,4,1e-05,0.9746444032158318,0.943,0.02184680361707013,0.5692502257120079,0.034771819647641575,0.43256772458393544,0.0233634075991481,0.17679272538606905,0.007668194919986859,0.9662452463366895,0.006148103128195363,0.878917378917379,0.02908252649809801,0.5470188356526116,0.0362099382523936,0.4138519074688882,0.027345791961117884,0.09082474692862029,0.005808831078849859,0.8958202632877043,0.008035621885511554,0.9400564537393836,0.009134905713481647,0.49234372072457055,0.014370784864646631,0.9647276478070979,0.0039017286602883216,0.9235177368548603,0.01012576780838096,0.9189732932079522,0.003530267429751703,,,,0.47597794758075834,0.03239373479241791,0.558581347045137,0.038240055904861195,0.2621098289071951,0.03934582654858117,0.40951543215189046,0.030112049427227196,0.8638400038292596,0.010322399053324752,0.375,2.8431650207494386e-05,0.14146341463414633,,,,,,,,,,0.006760732303569208,0.6146480741594899,0.006760732303569208,Baseline
+143,0,Pref. Opt.,Llama-2-7B,0.0,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9706077188940092,0.017620908359488253,0.5303743109742608,0.03407162338129851,0.9937714403974752,0.0010783212615616724,0.9681058114035087,0.018374271519515418,0.9166079435138229,0.008456020405592082,0.4,4.765788617078215e-07,0.08414634146341464,,,,,,,,,,0.003018184077228396,0.6285644571327744,0.003018184077228396,Baseline
+144,0,Pref. Opt.,Llama-2-7B,0.0,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5924105428404896,0.033263694434418234,0.4565522451578235,0.021610794050383905,0.1766362229531734,0.007145618092346133,0.9709098974802777,0.0050599919253484765,0.8874643874643875,0.02809493708284436,0.5594055257705869,0.03567260486346218,0.43350943486503823,0.02661437028896882,0.1056862156657845,0.006183210785840322,0.9198611150631801,0.006919614182981682,0.9810836592209436,0.005358997022471166,0.47823084373482894,0.013991136456899415,0.987682564962214,0.0014294465117274482,0.9837301765046863,0.005270907546412638,0.9240710251171401,0.003335151516141506,,,,0.9685243855606759,0.017780552723972488,0.5416122420147298,0.0335048502246019,0.9917345789108578,0.0011589828601444156,0.9660224780701754,0.018520210577111264,0.9166642498201292,0.008456217893549868,0.425,1.6943467926466838e-06,0.1,,,,,,,,,,0.0012708143485281624,0.6296293128306515,0.0012708143485281624,Baseline
+145,0,Pref. Opt.,Llama-2-7B,0.0,1,2,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.606546647433683,0.030961077053547004,0.46835196679220437,0.019662428798979843,0.18027710108089362,0.007120540722361764,0.9711053228188717,0.005193141445257983,0.8717948717948718,0.029276166353085355,0.5598299511583128,0.03513500053040345,0.4438246441699819,0.025308810405222027,0.10646149121607264,0.006189134075678301,0.9216962290636657,0.006709124114274743,0.9719699980936025,0.0063748268799224865,0.468221641983294,0.013605114756446606,0.9822887429450634,0.0014869938769852837,0.9847992127106778,0.0049558019543791566,0.9257162977066123,0.003308987703675799,,,,0.8186714818241565,0.043115858610446985,0.566483639868165,0.03249111598229366,0.9565788230001161,0.005529587923744388,0.8353519284034615,0.042068217012389,0.9176329318591485,0.008773384722766613,0.425,4.5276301534613594e-06,0.11341463414634147,,,,,,,,,,0.0012708143485281624,0.632871908926051,0.0012708143485281624,Baseline
+146,0,Pref. Opt.,Llama-2-7B,0.0,1,3,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.6209160392815353,0.02921498876116747,0.47751150900909095,0.018195830443283427,0.18402368903026795,0.007452606075757043,0.971230589018051,0.005180439432376149,0.8660968660968662,0.03016145359937715,0.5636513328534187,0.03397030619562475,0.4500346651850733,0.024116036374058982,0.10894654161447405,0.006392292787809994,0.9267264216214894,0.006596954642520662,0.9394917009335717,0.010209371403798576,0.460885816164811,0.013399740935308158,0.9751565902878809,0.0017260002980401803,0.9733056382182119,0.006991045991164258,0.9287424401721425,0.0033082966207093195,,,,0.5236425314354136,0.06006007523324811,0.5820412789879955,0.03188653465299932,0.9042981066202153,0.008310046664909112,0.5434620650095046,0.061323014422537105,0.9316512654147395,0.010667582742999213,0.375,5.845156920258887e-06,0.11707317073170732,,,,,,,,,,0.006760732303569208,0.6345567312241833,0.006760732303569208,Baseline
+147,0,Pref. Opt.,Llama-2-7B,0.0,1,4,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.6226602095796977,0.028556734993852778,0.479269047982605,0.017397555112613837,0.19010803523961417,0.007968879579021804,0.971887674038294,0.005207517121420166,0.8917378917378919,0.02724332888874447,0.5622231069115052,0.0336037536961641,0.45085733177400755,0.02340373364616858,0.11387149699825733,0.0061898143954831585,0.9334525928992325,0.006707866917734486,0.905155660984239,0.012720361050411409,0.4567800978942038,0.013256573510614282,0.9686896057092785,0.0019275416460468816,0.961450621340768,0.008536871990541558,0.9323639006302699,0.0032033723924312922,,,,0.15229945036720657,0.049092941234932366,0.5901997851872188,0.031438784846847985,0.8591139219396796,0.00928767343239399,0.18063505337112148,0.0493767152706705,0.9691722958986757,0.010434715487281379,0.375,1.6715837773517706e-05,0.1329268292682927,,,,,,,,,,0.006760732303569208,0.6335834527767225,0.006760732303569208,Baseline
+169,0,Retain Model,Llama-2-7B,0.0,1,-1,1e-05,0.0,0.945,0.02105188395801711,0.5926241084713598,0.03345059593050104,0.4544514898987734,0.021893655065203167,0.17353989749500584,0.006538369470852875,0.967232018904305,0.005690027028802709,0.878917378917379,0.02908252649809801,0.5442517457082237,0.03644994289779965,0.4220565650208274,0.027247431184912106,0.10057341165244862,0.006012069013665042,0.9135588025349304,0.006767434505349635,0.9809455309460164,0.0058307259379082295,0.47882616449120396,0.01410518223667099,0.990023753985948,0.0007292641153727538,0.9846272824265296,0.005002917384509446,0.9241821753906638,0.0033307787579352774,,,,0.4057248745903669,0.029331873414771226,0.6701244699861956,0.02808055457287809,0.1736882915925146,0.025915814128189336,0.4039820476083342,0.03168689004734824,0.8936834437422675,0.012238378133721525,0.0,1.0,0.0,,,,,,,,,,1.0,0.6246309817225171,1.0,Baseline
+148,0,Grad. Ascent,Llama-2-7B,0.01,1,0,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
+149,0,Grad. Ascent,Llama-2-7B,0.01,1,1,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5800258532067911,0.034048775490725,0.4479316827635683,0.02371490849542139,0.17520921205557424,0.007085165889905948,0.9712346794000724,0.004948616955739485,0.8831908831908832,0.02891603593003461,0.5605509089165185,0.03578007402333147,0.4380829002046114,0.0319031334543632,0.09930950443878336,0.006054405871120595,0.9114705966643226,0.00693546302693967,0.9812901725231946,0.005294521902057731,0.48294462031737356,0.01404737202686901,0.9880106777149297,0.0009239754509367609,0.9815029129733047,0.005299277163453248,0.9239376796665825,0.003345169932380108,,,,0.7977512522720149,0.038099059434458245,0.5338572177221842,0.032619997082725165,0.9015410916473158,0.017060016135887153,0.7871158272978711,0.03931892842575011,0.9099585668064352,0.010321529594744945,0.45,0.0011065220933232922,0.2121951219512195,,,,,,,,,,0.0005039436209702519,0.6280934372877442,0.0005039436209702519,Baseline
+150,0,Grad. Ascent,Llama-2-7B,0.01,1,2,1e-05,0.98989898989899,0.9329999999999999,0.02378587136868601,0.5743467636338636,0.03437254720889397,0.44299784489058774,0.023919808917236895,0.17549321155280903,0.007031472326137827,0.9716746149878339,0.004948396958023329,0.8874643874643875,0.0287431042820868,0.5528340897493834,0.03593856509215231,0.4337222544394661,0.03205121933349278,0.09969244858530825,0.006010696651478088,0.913665955907049,0.006868370009154204,0.9769308651638873,0.005956200936345651,0.4843319150944858,0.014081501536447579,0.9873622386393629,0.001058850510217853,0.9770827849479735,0.005975149733698831,0.9237527444463929,0.0033456930924993327,,,,0.6770552495067294,0.04141887944577475,0.5313505413009971,0.032377653394326515,0.7863983416805109,0.03208577061571647,0.6732327143425557,0.04134044820373354,0.9030402645531372,0.01113914903642642,0.45,0.0007984068197401939,0.2048780487804878,,,,,,,,,,0.0005039436209702519,0.6244505684885571,0.0005039436209702519,Baseline
+151,0,Grad. Ascent,Llama-2-7B,0.01,1,3,1e-05,0.98989898989899,0.9229999999999999,0.02553883950441608,0.5605770276733398,0.034814648533758916,0.4302945032709254,0.024239223157741678,0.17157705989401034,0.006932316728979308,0.9699656201319954,0.005618235030283827,0.8660968660968662,0.030766115083144873,0.5465426863610293,0.03605363485526744,0.4271626730893178,0.032234933455384716,0.09585526374650297,0.006212504348587216,0.9066163830397892,0.007232014686092795,0.952820588478855,0.008377862576579216,0.48822552142839126,0.01408030508332057,0.9795794731504118,0.0020893988669301346,0.9511125315484827,0.008266508391635445,0.9215150317951316,0.0034707050691802194,,,,0.5415403816676896,0.03633105352195194,0.536498562129631,0.034032965640264846,0.45397324031395614,0.041194928318438455,0.5190002196868267,0.036133109171101345,0.8924787780443587,0.013457525433243911,0.425,0.0009936344267771346,0.2097560975609756,,,,,,,,,,0.0012708143485281624,0.6149556371319791,0.0012708143485281624,Baseline
+152,0,Grad. Ascent,Llama-2-7B,0.01,1,4,1e-05,0.98989898989899,0.9229999999999999,0.02553883950441608,0.557230500186014,0.034865896707917045,0.4258273633007778,0.024371014502785608,0.1694671556041842,0.006915481170274377,0.9687123356509246,0.006122574201166815,0.8618233618233619,0.030902274103896878,0.5440057886627606,0.03608321893084478,0.4248389901132918,0.03233250572807769,0.09186920722354087,0.006054334602349437,0.9016770263139969,0.007509414988868026,0.9414179134529468,0.009380750631925328,0.4891483824586931,0.014149859802891799,0.9757943241994819,0.0024221258047436266,0.9382342689565394,0.009331698800946149,0.9206883117273208,0.0035053740277021716,,,,0.515438355924935,0.03450413884417871,0.5349858627805251,0.03441539018972547,0.3484092705229157,0.03912387599627555,0.4880769534734265,0.03613318624504579,0.8743174909059739,0.012123663709260601,0.4,0.0013674832061951747,0.21707317073170732,,,,,,,,,,0.003018184077228396,0.611821828661174,0.003018184077228396,Baseline
+128,0,Grad. Diff.,Llama-2-7B,0.01,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
+129,0,Grad. Diff.,Llama-2-7B,0.01,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5773765341443244,0.03417794235272885,0.4441402807130932,0.02402033482691025,0.17514580487452006,0.007137633259603525,0.9715280541412732,0.004854702506416361,0.8746438746438747,0.02986595450792445,0.5548336070164175,0.03585489922508421,0.4340930797609021,0.032090674904577,0.10036751772585996,0.0062275981311647145,0.9128550837341167,0.006846382858297305,0.9812212692042912,0.005429907335500562,0.4828983996735526,0.014174579326499147,0.9889669057437027,0.0008745702996192081,0.9769337794228795,0.006224395582782556,0.9225404481429507,0.003328455303415582,,,,0.8390636054175344,0.03505917880239604,0.5313141361531593,0.03296334226008337,0.9427974085006843,0.011427898495431734,0.8358652145198164,0.03613972046757268,0.9157937185986256,0.010161506733134736,0.425,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.0012708143485281624,0.6247498581220056,0.0012708143485281624,Baseline
+130,0,Grad. Diff.,Llama-2-7B,0.01,1,2,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5692104903614713,0.03439893577976867,0.4361785377749471,0.02410946926548582,0.17353800989172505,0.007043791207283828,0.9731467522708408,0.004779718421559819,0.878917378917379,0.029709155181433657,0.5485601874923083,0.036093590198850146,0.42982827628427805,0.032163414601145704,0.09954316594928822,0.006028171804746514,0.9099425999261167,0.007378812492078975,0.9822425534755755,0.005285317705376488,0.48467084926549414,0.01420301034167578,0.9879979225459306,0.0010270337380533711,0.9772090305588494,0.006242996332846527,0.9224518287464465,0.0033328897682729734,,,,0.7215835454295053,0.043592000438350396,0.5276723573440643,0.03284856684429458,0.8300386090545793,0.028438250445988612,0.7318984963117551,0.043410059838410925,0.9098268541289307,0.010555482868827449,0.45,0.0007553515715699177,0.20365853658536584,,,,,,,,,,0.0005039436209702519,0.6206049086240365,0.0005039436209702519,Baseline
+131,0,Grad. Diff.,Llama-2-7B,0.01,1,3,1e-05,0.9837146980004123,0.9229999999999999,0.02553883950441608,0.5574112343706722,0.03481966255137871,0.4228619983745002,0.024639522296912447,0.16625247534991444,0.006622426447691064,0.9726272150051648,0.005160253129210175,0.8660968660968662,0.030766115083144873,0.5414314892361967,0.0361188042720218,0.42204842880404664,0.0325687219555866,0.09611545317402165,0.00602262255391896,0.9086252009559641,0.007404644959587136,0.9550050353791786,0.008610146710774336,0.4872969147494578,0.014309154781936713,0.9810517654599463,0.0018522824870059965,0.9479980529273969,0.00914181826143523,0.9208812427369228,0.0033613659941016034,,,,0.5765635074871464,0.03586214752395286,0.5253032114387033,0.03385578852277948,0.5372222789151637,0.04213828810721073,0.5501486653771084,0.035514857396460875,0.8946132097747274,0.011117196442996429,0.425,0.0005381573264457984,0.19634146341463415,,,,,,,,,,0.0012708143485281624,0.6109236267561884,0.0012708143485281624,Baseline
+132,0,Grad. Diff.,Llama-2-7B,0.01,1,4,1e-05,0.9837146980004123,0.9179999999999999,0.02683206283459391,0.5520435933650645,0.03481328532455935,0.415991757729618,0.024877486524841206,0.1653701412476612,0.006760408232245415,0.9722346052249344,0.005086449152721892,0.8660968660968662,0.030766115083144873,0.5371617229622194,0.03607485029885521,0.4187476835115922,0.032689354178344066,0.09600424215333588,0.00599081988604742,0.9103354497720064,0.00732678373911513,0.9400970491453476,0.0097824948523407,0.48856732664519475,0.014411133751347567,0.9759288396427315,0.0021738910844479944,0.9302713108973918,0.01033028178851249,0.9200524749123263,0.0033835639324742406,,,,0.5084746164474694,0.03774565816012248,0.5240481718915301,0.035010545007557396,0.40058082944838047,0.04163737641558911,0.5073350257593472,0.03910212571406874,0.8901339308097882,0.011020560726255227,0.425,0.0008912169196264585,0.2073170731707317,,,,,,,,,,0.0012708143485281624,0.6062986298187203,0.0012708143485281624,Baseline
+158,0,KL Min.,Llama-2-7B,0.01,1,0,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
+159,0,KL Min.,Llama-2-7B,0.01,1,1,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.580110523651442,0.0342139237242958,0.44809062095334956,0.023725648301852434,0.1755352610066232,0.007067444719009651,0.9712346794000724,0.004948616955739485,0.8874643874643875,0.0287431042820868,0.5598849904079047,0.03575319998562076,0.4386095190114807,0.03188548587354564,0.0995082966488098,0.006022209617325384,0.9111060994449297,0.006978441012047056,0.9794719907050128,0.005675897883214464,0.4827531596223028,0.014049862186615527,0.9880365938733333,0.0009178672836727823,0.9804534198307884,0.005435411049767168,0.9242813685944077,0.0033437399379373286,,,,0.7950726808434434,0.03891439833031167,0.5335169256291284,0.03239227876815981,0.9014313140700427,0.017028902783455285,0.7837473656468008,0.04010397898938578,0.9104373329081868,0.010183776798575905,0.425,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.0012708143485281624,0.6282876063598299,0.0012708143485281624,Baseline
+160,0,KL Min.,Llama-2-7B,0.01,1,2,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5754402690990706,0.03430368893520067,0.44406569523416695,0.023922200543703864,0.17622850567045614,0.007070840040822426,0.9734853810333826,0.0048032052840292685,0.8831908831908832,0.02891603593003461,0.5539892135948579,0.036011802420717914,0.43446712016321604,0.0320295895982226,0.09881985331421088,0.006023363827002265,0.9090761817520574,0.0072184737616799765,0.9773090164243915,0.005847060879197442,0.484487386660019,0.014074711958540534,0.98738023909771,0.001054780870647492,0.9764651020476818,0.006139645378646104,0.9233519772587567,0.00336247411780429,,,,0.6796703091782407,0.04243778859675377,0.5311777081329236,0.03249675958188966,0.786087349932698,0.03206775939949202,0.6741205310232469,0.04218301837400526,0.8993946675439609,0.011606517481817501,0.45,0.0009411722930963151,0.20853658536585365,,,,,,,,,,0.0005039436209702519,0.6249746605306692,0.0005039436209702519,Baseline
+161,0,KL Min.,Llama-2-7B,0.01,1,3,1e-05,0.9746444032158318,0.9329999999999999,0.02378587136868601,0.5613741848967867,0.03464862221361874,0.43026663892093714,0.02419244482979826,0.17626374559393135,0.007117271946580789,0.9703688320138824,0.005590378744295158,0.8660968660968662,0.030766115083144873,0.5470421743758249,0.035915415061641885,0.426414986253531,0.032244399064120764,0.09518918609090538,0.006247104685379855,0.9064787748921043,0.007343742226863473,0.9530836815948891,0.008434206671982544,0.48817648773815203,0.014081933722265658,0.9797252028178876,0.0020761009586852067,0.952067666485085,0.008289357264083934,0.9216562188281612,0.003495696036059303,,,,0.5417339802648964,0.03560176457222332,0.5359991224635998,0.033713459909138134,0.45529475231527805,0.0412123298354884,0.5222129245706866,0.035585566289791426,0.8883371883219329,0.013243955948083337,0.425,0.0012308068326092325,0.2146341463414634,,,,,,,,,,0.0012708143485281624,0.6154517140995398,0.0012708143485281624,Baseline
+162,0,KL Min.,Llama-2-7B,0.01,1,4,1e-05,0.9746444032158318,0.9229999999999999,0.02553883950441608,0.5589617571264943,0.034722005379876165,0.42686555147068356,0.024509365403373253,0.16860481894266724,0.006869397681122072,0.9711684239097478,0.005048161498283998,0.8618233618233619,0.030902274103896878,0.544318030740605,0.03604717566868592,0.4243098312756541,0.03234995486337138,0.0916169728987259,0.006034686543439203,0.901223023759475,0.007686433410408507,0.9430402910753245,0.00937070133163249,0.48913036359507317,0.01417050329800232,0.9761569459467297,0.002351134271672225,0.9412342727443869,0.009122881887916547,0.9206378021156858,0.0035345532864452995,,,,0.5197038216420136,0.03425461287449973,0.534672398512898,0.034361945689757485,0.3493818427001489,0.03916070926834415,0.48833168881585126,0.036106213139953505,0.8753322814923875,0.011951847964291485,0.4,0.0013674832061951747,0.21707317073170732,,,,,,,,,,0.003018184077228396,0.6123014101651115,0.003018184077228396,Baseline
+138,0,Pref. Opt.,Llama-2-7B,0.01,1,0,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5784785897528664,0.03427796265180226,0.4467510220078141,0.02404774633564275,0.176106843835559,0.007126903653416667,0.9719848047133556,0.004864797872712978,0.8874643874643875,0.0287431042820868,0.5578633007430241,0.035865385073650724,0.43629045271172606,0.032037037914719325,0.1010033466750109,0.006193486249553319,0.9122018079375641,0.006939964994785374,0.9831651085190984,0.005299978907009621,0.4823950208724669,0.014162541476855167,0.9895421485384123,0.0008054492074133557,0.9834026198552906,0.005309710112001442,0.9239805127970684,0.003321195499902465,,,,0.950667242703533,0.024753643396748466,0.5346892802341167,0.033672810564330136,0.9930274019397498,0.0010067000192630349,0.9479979236753575,0.025217356544835114,0.9177029237579706,0.00873151308921818,0.425,0.0012975311765330844,0.21585365853658536,,,,,,,,,,0.0012708143485281624,0.6271357825103671,0.0012708143485281624,Baseline
+139,0,Pref. Opt.,Llama-2-7B,0.01,1,1,1e-05,0.9837146980004123,0.9329999999999999,0.02378587136868601,0.5818305995434252,0.0336258806698115,0.4516876545117113,0.022786998297976563,0.17735101469737688,0.007285532670262744,0.9709609804468069,0.004962772317491594,0.8746438746438747,0.029865954507924442,0.559317079149142,0.03573241714959832,0.4403488213603541,0.031356894081744115,0.10132621009432234,0.006328051690957193,0.912779227727123,0.00708526437373145,0.9809739936408115,0.005541810173179431,0.4769229838409399,0.013995921593912948,0.9884367518383954,0.0008572436618283804,0.9868234582761289,0.0044571911882983406,0.9248143893891777,0.0033181315019821128,,,,0.9624071550335319,0.01902751362238947,0.5445679585904102,0.03260085787341239,0.9902187818386323,0.00128667594293562,0.9679016436903499,0.018337879111251044,0.9172507689793704,0.008676551693240605,0.425,0.001770566779669025,0.22317073170731708,,,,,,,,,,0.0012708143485281624,0.6278577940849149,0.0012708143485281624,Baseline
+140,0,Pref. Opt.,Llama-2-7B,0.01,1,2,1e-05,0.9837146980004123,0.9363333333333332,0.0226726819112893,0.5868525414917711,0.03227039377073378,0.4549850082151925,0.021227661982427308,0.17970599305235527,0.007227741865684971,0.9730753101674295,0.004857974780534291,0.8703703703703705,0.03001669034410197,0.554188936629353,0.03570215328899892,0.4434825705074438,0.030519104128676356,0.1040392353475466,0.006307929113897188,0.9189298043221829,0.007206825363839335,0.9703738069804334,0.00655653596424914,0.4711610329750893,0.013811135050113365,0.9861218016375611,0.0009899238404132483,0.9878782368309074,0.004284678608913816,0.9267550927503032,0.003260527718965425,,,,0.8363801365883157,0.04517391821008251,0.5571474547164842,0.03192519282255046,0.972944523546138,0.0039513920926697995,0.8405939756050049,0.043694964587067335,0.9191168798699401,0.009173316675796253,0.4,0.002638970910993521,0.2329268292682927,,,,,,,,,,0.003018184077228396,0.6273896698700678,0.003018184077228396,Baseline
+141,0,Pref. Opt.,Llama-2-7B,0.01,1,3,1e-05,0.9837146980004123,0.9263333333333332,0.024519146644786646,0.6120984951453181,0.029602092247205808,0.4701069849132854,0.018763352188774993,0.179398180165208,0.007462407405966698,0.9719109744898553,0.005046110224429372,0.8746438746438747,0.02924268600158814,0.558692577179061,0.03470032908926651,0.45270895342466627,0.028971933358048183,0.10814599375745441,0.006333302982646953,0.9223662570782941,0.007279004718073149,0.9488138305990799,0.009263087395909031,0.46129806944852114,0.013487744052917184,0.977738637105947,0.0015008608589127987,0.980706964350727,0.006084155109445612,0.9294579680297151,0.0032532959867766285,,,,0.6071157085321186,0.055098423683929434,0.5789698666797543,0.031504356391908586,0.9111425180364293,0.008094541326696002,0.6247685301405814,0.05693647789581892,0.9083199808994191,0.011159154252143345,0.4,0.006955192886380246,0.25853658536585367,,,,,,,,,,0.003018184077228396,0.6325965973075246,0.003018184077228396,Baseline
+142,0,Pref. Opt.,Llama-2-7B,0.01,1,4,1e-05,0.9837146980004123,0.9263333333333332,0.024519146644786646,0.612985074642076,0.029414999552961535,0.47193884066480796,0.018208375192243286,0.17961196637899424,0.007417501989608266,0.972482403061284,0.0050452315349812996,0.8660968660968662,0.030161453599377153,0.5611374865952636,0.03433914339069163,0.4553333687005546,0.02848497315421883,0.10880235579883603,0.006353917645501141,0.9243275141232049,0.007109171912797723,0.926926400731736,0.011283896071661435,0.45949966474157417,0.013422309069073818,0.9747282014283064,0.0016379979235600203,0.9758053680543032,0.006949235339699891,0.9309483108631779,0.0032548366189807136,,,,0.3999643294452556,0.06221025434190201,0.5839567149240932,0.03139728091889601,0.8851536619883351,0.008884138981633464,0.4390431857167497,0.06400171941745215,0.9295107228173798,0.01018216034602813,0.4,0.007917965558590367,0.2621951219512195,,,,,,,,,,0.003018184077228396,0.63185656470058,0.003018184077228396,Baseline
+168,0,Retain Model,Llama-2-7B,0.01,1,-1,1e-05,0.0,0.925,0.02487849259781215,0.5862944163094279,0.03374892372874021,0.4505683636462643,0.022065610809776242,0.16783975493031222,0.006529491837121361,0.9700812534017699,0.005269946117413013,0.8703703703703705,0.03001669034410197,0.5471365216754651,0.03632183056706994,0.42191312801444286,0.027224296551869066,0.09780821686211255,0.006152149197443829,0.9107498232250473,0.007020623478948183,0.9852486009202834,0.004927942762345694,0.4782308271896145,0.014196414741213824,0.989928497616268,0.0007514551842063065,0.9870143086012513,0.0044681490010742096,0.9240008227178458,0.0033281752786762583,,,,0.40618862927483795,0.030258984159567487,0.6701777021800741,0.028498124312231716,0.17508310272500335,0.02529203048185361,0.3914341294336592,0.028405449629408948,0.8923673878445355,0.012032068634356475,0.0,1.0,0.0,,,,,,,,,,1.0,0.6220150094809674,1.0,Baseline

versions/llama-5p/llama-5p.csv

@@ -0,0 +1,44 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+254,0,Finetune Model,Llama-2-7B,0.0,0,-1,1e-05,0.0,0.9329999999999999,0.02378587136868601,0.5906712782297949,0.03387326041191319,0.4529615680881898,0.022529558405746858,0.17365584961134498,0.007052746400281119,0.9711479727132513,0.004934130913988655,0.8831908831908832,0.02891603593003461,0.5596387660002274,0.03594771311341075,0.4279401568170806,0.027010465124066053,0.10041107389135993,0.006099955540692599,0.9116643120381124,0.007061923106032409,0.9885927710282548,0.004116614633043364,0.48376209606309467,0.014166403314249702,0.9888492808338375,0.001496862772045587,0.9839221638885342,0.005172697629831861,0.9228793357882765,0.0033391564038546253,,,,0.9844665453461495,0.0051727714413351816,0.5021345577886491,0.013075097432894608,0.9905248646281772,0.0007461848129786882,0.9876392022245997,0.004309388867036841,0.9158566386007846,0.0035939738499808516,0.38,4.709184258054772e-44,0.036345514950166115,,,,,,,,,,1.096624314778916e-19,0.6285644571327744,1.096624314778916e-19,Baseline
+195,0,Grad. Ascent,Llama-2-7B,0.0,5,0,1e-05,0.9473684210526315,0.9329999999999999,0.02378587136868601,0.6016525260655208,0.03296290140542491,0.4658663986798423,0.02138104832314672,0.1796917205601416,0.006957842351463092,0.9726675125714572,0.005003661205010963,0.8960113960113961,0.027048070647329614,0.5673914574330066,0.035699599342913266,0.44606244685946644,0.026310490137637428,0.10471484259468007,0.005993892351347259,0.9189366034130854,0.007022782507540524,0.9624338146058092,0.007215035751130736,0.4776503795319775,0.013705366204408343,0.9748034396170439,0.0023680799124731135,0.9585553399470043,0.007405881460470756,0.9216893870198567,0.003519956535532923,,,,0.8325535359780731,0.017908284586238565,0.5149809301660234,0.014639723450722002,0.9177597787162979,0.007549678376520792,0.8439743162020424,0.01740344305201923,0.9084408432141143,0.0050946291125713574,0.425,4.817451537281918e-28,0.052338308457711445,,,,,,,,,,1.3969162950232394e-16,0.6356269898634712,1.3969162950232394e-16,Baseline
+196,0,Grad. Ascent,Llama-2-7B,0.0,5,1,1e-05,0.9473684210526315,0.9363333333333332,0.0226726819112893,0.48716785454669626,0.0368579986247528,0.3720096278953561,0.022934960621886612,0.17037853482946363,0.006423502492282719,0.9699184988936387,0.00511356240415103,0.8917378917378919,0.02724332888874447,0.5161490168877112,0.037156340447104655,0.3985509454432953,0.027779582706722688,0.09513363076776075,0.005519286159899693,0.900314904832396,0.008325995283480414,0.5671224696227922,0.015085026888310255,0.48035178431942815,0.014228702453558664,0.5489780913142936,0.013454140409973833,0.45295299504150727,0.015332082150668565,0.8482062491318045,0.006687152082176721,,,,0.44418601407195196,0.013711819079717028,0.5864884351477357,0.015492677347034836,0.21641887531880108,0.013305780804320239,0.33251308739332885,0.012805236505189433,0.7926682461459557,0.010033495992984384,0.27,1.2432373778250906e-10,0.23766169154228856,,,,,,,,,,8.056669472003223e-07,0.5280648719498976,8.056669472003223e-07,Baseline
+197,0,Grad. Ascent,Llama-2-7B,0.0,5,2,1e-05,0.9473684210526316,0.015,0.01113507516768068,0.5460218390127364,0.042978933651256615,0.3673814300957872,0.03288261543522096,0.00016584429824561403,0.00011687491196819323,0.08339422139770254,0.0031688307588229867,0.5081196581196581,0.04404180244422612,0.5760455377319045,0.039696328950134155,0.39963813066542275,0.032938923039781916,0.00848662775791482,0.0013757783199858148,0.12372907543836983,0.0063853963698491814,0.05291640350105928,0.00450905030611381,0.49879081689258875,0.021478355183181145,9.069113266767572e-05,4.127835721755367e-05,0.008809878625602861,0.0011092617938764262,0.043256883129473865,0.0017559052778549152,,,,0.020595613486419282,0.00350775587935582,0.5233024802354518,0.020794867990479315,1.7064736214079959e-06,6.91278989826657e-07,0.003979891916884827,0.0008158201394767412,0.0275549869029456,0.001981155969927457,0.18,0.8768515798873443,0.493681592039801,,,,,,,,,,0.0030098470801667434,0.0008090058350146668,0.0030098470801667434,Baseline
+198,0,Grad. Ascent,Llama-2-7B,0.0,5,3,1e-05,0.9473684210526315,0.0,0.0,0.31637098954621684,0.036558172855651654,0.26654307001348143,0.025562005078696985,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.28343098718321885,0.0318935499910344,0.2584320192221162,0.0207683512147745,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.000726911976911977,0.00032581928830398776,0.11581094708121663,0.010347975899324065,8.720964580269228e-40,1.3506647400115664e-40,0.0001029238225131973,4.5729592356619765e-05,0.00601575039373741,8.670964358060814e-06,,,,0.002577353254161311,0.0006578035123613063,0.7650603882001618,0.013321611374627583,2.5734877176144837e-39,1.2872214671608142e-39,0.0004730642929160991,0.00011796711278424548,0.00627147628227061,1.7349393200286088e-05,0.395,4.996724050090626e-10,0.2464179104477612,,,,,,,,,,2.5656301025789152e-14,0.0,2.5656301025789152e-14,Baseline
+199,0,Grad. Ascent,Llama-2-7B,0.0,5,4,1e-05,0.9473684210526316,0.0,0.0,0.34995885651667413,0.03900023681247861,0.2597072740494202,0.027315488037385767,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.3168197879447783,0.03354331078849264,0.2573087574172523,0.023564907966181322,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.000726911976911977,0.00032581928830398776,0.11332344276777126,0.010606504668061721,7.5301974036364085e-50,1.2542477594190219e-50,0.0001029238225131973,4.5729592356619765e-05,0.00601575039373741,8.670964358060814e-06,,,,0.002577353254161311,0.0006578035123613063,0.748459462506477,0.014292059912097893,1.7819523672620854e-42,5.853666664296551e-43,0.0004730642929160991,0.00011796711278424548,0.00627147628227061,1.7349393200286088e-05,0.365,4.163584010208238e-09,0.26034825870646766,,,,,,,,,,3.0799540915748314e-12,0.0,3.0799540915748314e-12,Baseline
+175,0,Grad. Diff.,Llama-2-7B,0.0,5,0,1e-05,0.9151450053705693,0.9329999999999999,0.02378587136868601,0.5964608906104591,0.033736886510858585,0.4596732035440541,0.02219720676920561,0.17589589164705263,0.007067505481473189,0.9726314166482165,0.004851474221742234,0.8888888888888888,0.027961373813112016,0.563026747464287,0.03587684372054116,0.43433219516638366,0.02684862457967057,0.1039989283221901,0.006011628764834727,0.9191355400524177,0.006766298730911742,0.9766369007098221,0.006086808464048844,0.480614966744523,0.014042304705377621,0.9830555795153016,0.0019108436473557276,0.9698792010821421,0.006955343867417926,0.9227607968919845,0.003358810582204389,,,,0.8866666349592291,0.015599383650741724,0.5074151875309192,0.015044240516810152,0.9483499199609132,0.005989474635820105,0.8863028979507425,0.016027550739032906,0.9087820617506792,0.004598714262661703,0.405,1.3085099009794603e-28,0.04756218905472637,,,,,,,,,,4.734741877153768e-15,0.6316165529510213,4.734741877153768e-15,Baseline
+176,0,Grad. Diff.,Llama-2-7B,0.0,5,1,1e-05,0.9151450053705693,0.9063333333333332,0.02773813332587542,0.5170184376811758,0.036619386725179594,0.3857944922017707,0.0241007072086274,0.1678010157907183,0.007119409867864278,0.9719120662079018,0.005267685983943506,0.8831908831908832,0.028916035930034605,0.5285941212999701,0.03673716302231243,0.3990152166248107,0.028257106198486807,0.10715362863015915,0.006140247979800844,0.9233003938202665,0.007375528710063154,0.7021353704084803,0.016476887627020258,0.4832227023973624,0.015154103719288615,0.827790691754942,0.01100572196227861,0.7176729587205578,0.0157190083605284,0.9198747516959845,0.0037884789778447328,,,,0.4842424307522695,0.016202853290665516,0.5218512859247023,0.01655349525411143,0.35483599699955504,0.019481630750004888,0.4975852178542688,0.01592683996314727,0.8830586520754045,0.007769772827416865,0.36,7.526258864342888e-16,0.17129353233830846,,,,,,,,,,6.5689645859369454e-12,0.5674114046866653,6.5689645859369454e-12,Baseline
+177,0,Grad. Diff.,Llama-2-7B,0.0,5,2,1e-05,0.9151450053705693,0.7303333333333333,0.043301192431540104,0.5039981076741162,0.038393227463920786,0.3457764520046886,0.027605361066219997,0.12485783257312966,0.009246795447667967,0.9185245787019578,0.017034627067647308,0.8774928774928775,0.028353390783834,0.5590056247760194,0.03836428044987644,0.3988147663540158,0.031244237597116486,0.0989169183994494,0.005897960692012901,0.9062810146518292,0.00827006541520969,0.3388438550685795,0.013316997144861044,0.49392829781245684,0.01794719021570066,0.187093283263421,0.012891498032795707,0.33438871430246714,0.013796448557085263,0.709042697553905,0.017589526839071404,,,,0.2245934627954196,0.014527792014854529,0.4921434666649202,0.019977571834798046,0.009406881800780927,0.0024256739081716097,0.22239844990991017,0.015460497105858189,0.5080743832765368,0.02734735162303483,0.205,0.27642217820098325,0.4556218905472637,,,,,,,,,,0.0004306111484986726,0.40905242908964196,0.0004306111484986726,Baseline
+178,0,Grad. Diff.,Llama-2-7B,0.0,5,3,1e-05,0.9151450053705693,0.16333333333333333,0.03553187342627315,0.7659694703250493,0.03511942882517864,0.5648246945456754,0.03307119617059185,0.03378077478077478,0.008223726781771547,0.14278311009328026,0.027956077362981897,0.7279202279202279,0.039574308467940814,0.6582486845068921,0.036262164710163795,0.5082570025049761,0.03299433556014912,0.07572768895833235,0.006210294557650443,0.49411309505711437,0.03415085224498574,0.11525995369034069,0.009850372917199327,0.624089886968335,0.020305308949506825,0.07958082123165239,0.00855926470119776,0.25205945513276745,0.017063173401762034,0.08405617971302116,0.008775255840420226,,,,0.001709332781913427,0.0006414161236408227,0.4782250606857153,0.021010705974621496,7.513866994215956e-15,7.496422953485618e-15,0.0009532563025210083,0.0007191103758319942,0.006672400996799395,0.00016892626899625522,0.3,0.006413415822221693,0.3888557213930348,,,,,,,,,,2.4125498977354686e-08,0.2438817635887742,2.4125498977354686e-08,Baseline
+179,0,Grad. Diff.,Llama-2-7B,0.0,5,4,1e-05,0.9151450053705693,0.191,0.03822038696557087,0.8213013620108808,0.03061657361565882,0.6357807794551209,0.03212521577724934,0.03726242023729174,0.008187174959018004,0.11775056661246898,0.027058932170038706,0.6794871794871795,0.0419129593837044,0.7022920875984318,0.03409313158581102,0.552441388702404,0.030761968419590394,0.08239508961061245,0.006893654255447367,0.3685683852488236,0.03675184117265645,0.4268563869874879,0.016601789178393795,0.5114804049601933,0.016792918798536345,0.574989003333673,0.01419354228328432,0.4936467351315724,0.01691980700444858,0.5252094874364516,0.02281665404143069,,,,0.0,0.0,0.5227158695303888,0.020399383299433783,3.030462968168194e-36,3.030462968168194e-36,0.0,0.0,0.00627147628227061,1.7349393200286088e-05,0.3,0.00010704568565310918,0.3420398009950249,,,,,,,,,,2.4125498977354686e-08,0.479519556937575,2.4125498977354686e-08,Baseline
+205,0,KL Min.,Llama-2-7B,0.0,5,0,1e-05,0.8678839957035446,0.9329999999999999,0.02378587136868601,0.6003655507792398,0.03307373017779353,0.4652067091345969,0.021341305728492757,0.1792711688895899,0.006954917494057758,0.9710458909498356,0.005174057301061736,0.8960113960113961,0.027048070647329614,0.5690945112311392,0.03571003687571289,0.4459749736282013,0.026311972666333498,0.10309056237195711,0.005899022232912953,0.9154073535131,0.007272660200452551,0.9633311576009784,0.007202588708663203,0.4794876645002475,0.013679355697746812,0.9752690142104361,0.0023565326317126997,0.9591493438369548,0.007482964964281113,0.9211836885950078,0.00354313220014963,,,,0.8311297718340204,0.018055861278960722,0.514177150320512,0.014665250285509628,0.9179190078949765,0.007562064674966894,0.8432933780349708,0.01735775161706432,0.908983539801799,0.0048218768751264185,0.425,3.389210472525946e-28,0.05104477611940299,,,,,,,,,,1.3969162950232394e-16,0.6359731727379991,1.3969162950232394e-16,Baseline
+206,0,KL Min.,Llama-2-7B,0.0,5,1,1e-05,0.8678839957035446,0.9263333333333332,0.024519146644786643,0.47638235723024946,0.037466228148907085,0.36337334048746556,0.02343695052213198,0.1688855115129109,0.00676703950837788,0.9700945839234856,0.005135234007347826,0.8960113960113961,0.027048070647329614,0.5139720117441711,0.03716339283306779,0.396210286817334,0.02796241633282945,0.096494589462537,0.005570998107069346,0.9042752853260453,0.008158447801148069,0.5695686786776466,0.01528305222433511,0.4827912120425598,0.014480067935742243,0.5852564071794397,0.014024143893775095,0.4695338809467642,0.01538284394404524,0.8592668087996913,0.006213915429351408,,,,0.4435843261954046,0.013872918567865296,0.5750351025210818,0.015454034113645142,0.2125491083007152,0.013772127629773647,0.3628371934360372,0.013328405498537354,0.8176807755830829,0.00899095299365914,0.275,8.71884956540331e-11,0.2354726368159204,,,,,,,,,,4.6128382074998843e-07,0.5277996009782985,4.6128382074998843e-07,Baseline
+207,0,KL Min.,Llama-2-7B,0.0,5,2,1e-05,0.8678839957035446,0.0,0.0,0.540348910718924,0.042285529636801364,0.3695139393164958,0.0324358745658412,0.0,0.0,0.06856250345723267,0.002911722649786752,0.3293447293447293,0.0410934883751986,0.583726240644272,0.03940166536268993,0.41360225065367984,0.033512353115342605,0.007941459620267877,0.0019018895364169904,0.09930221605110215,0.004092428182678375,0.029706990354132722,0.0030816303166996675,0.4623101177717977,0.020856594478691825,1.9580353552869956e-05,9.530654377193728e-06,0.005328143111488564,0.0009532693718812542,0.032386262994446254,0.0013138217753751779,,,,0.011577732688268647,0.0022333409416275232,0.5164737756124184,0.019684429595934576,6.430294485853603e-07,4.030919037815218e-07,0.0050775484310806715,0.002946652300095565,0.02375844138710737,0.0016412021813395012,0.205,0.7061493883552997,0.4846268656716418,,,,,,,,,,0.0004306111484986726,0.0,0.0004306111484986726,Baseline
+208,0,KL Min.,Llama-2-7B,0.0,5,3,1e-05,0.8678839957035446,0.0,0.0,0.32648623564267515,0.036376857329560766,0.26908456247808576,0.026073448468141684,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.294114829822957,0.03271973010320772,0.266836541452832,0.022239636581636024,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.000726911976911977,0.00032581928830398776,0.13133711730188175,0.01079409711205999,8.406208074029834e-39,1.2111254931040853e-39,0.0001029238225131973,4.5729592356619765e-05,0.00601575039373741,8.670964358060814e-06,,,,0.002577353254161311,0.0006578035123613063,0.7503216016446621,0.013977415921075766,8.825752062576762e-38,2.931174635121489e-38,0.0004730642929160991,0.00011796711278424548,0.00627147628227061,1.7349393200286088e-05,0.385,7.64645719429505e-10,0.2491542288557214,,,,,,,,,,1.3261055712130201e-13,0.0,1.3261055712130201e-13,Baseline
+209,0,KL Min.,Llama-2-7B,0.0,5,4,1e-05,0.8678839957035446,0.0,0.0,0.356170914514647,0.038929644820495005,0.26203117884374505,0.028189162320959825,0.0,0.0,0.00593927296691946,7.832918918323452e-06,0.0,0.0,0.3033755788682566,0.033323209409376754,0.2566912879551799,0.02323289367727593,0.0,0.0,0.0058544212256097915,1.5608478049323096e-05,0.000726911976911977,0.00032581928830398776,0.1232697528518183,0.010928407034098562,4.345721876813726e-49,5.474119652958691e-50,0.0001029238225131973,4.5729592356619765e-05,0.00601575039373741,8.670964358060814e-06,,,,0.002577353254161311,0.0006578035123613063,0.7445890390333506,0.01449717637287885,3.4832907818275815e-43,1.046547066792983e-43,0.0004730642929160991,0.00011796711278424548,0.00627147628227061,1.7349393200286088e-05,0.375,4.133013785694094e-09,0.26029850746268657,,,,,,,,,,6.540866008411473e-13,0.0,6.540866008411473e-13,Baseline
+185,0,Pref. Opt.,Llama-2-7B,0.0,5,0,1e-05,0.9151450053705693,0.9363333333333332,0.0226726819112893,0.6390164100486684,0.028618272096662102,0.49245941195501,0.018316664340797935,0.18501773707888258,0.007522194073289981,0.9703420638764966,0.005278075924439676,0.8746438746438747,0.029242686001588143,0.5782961595992523,0.03357761360489438,0.4613117281496306,0.02390978786965543,0.11071619534491209,0.006329757770999674,0.9296421791321634,0.006505138724513489,0.9235221576984062,0.011331939666996551,0.4608465725720934,0.013356097786578099,0.9728020922720532,0.0018547302653328262,0.9755858243481406,0.006625634007999791,0.9314239636930028,0.0032703856741554374,,,,0.9381706365356156,0.012077872257370892,0.5374386933378453,0.014505258988847322,0.9691933755282577,0.001970571703739957,0.9700325883540987,0.007893985305105078,0.9209143784306513,0.004469277648453111,0.4,1.2486447293815883e-21,0.11044776119402985,,,,,,,,,,1.108718722900174e-14,0.6435703224701355,1.108718722900174e-14,Baseline
+186,0,Pref. Opt.,Llama-2-7B,0.0,5,1,1e-05,0.9151450053705693,0.015333333333333332,0.010672346972385687,0.5794152120422705,0.0226247364548374,0.44686670772156023,0.012849379186332383,0.006190476190476191,0.0036701912683802227,1.0,0.0,0.5384615384615384,0.045121964130070795,0.5358066939015832,0.030711091939061654,0.43791934314206377,0.018090877372725626,0.0788137204813196,0.008943159064813532,0.9587586903776453,0.005959005396879436,0.021691219597878248,0.006130815027789931,0.415042664194484,0.012319151743399957,0.8072387564727065,0.0055942408307084425,0.05126984126984127,0.0075262717309817935,0.998695582295641,0.0008084337779245585,,,,0.011147538374262014,0.001320943721533609,0.6120681492866694,0.01392673798203847,0.7139919391570153,0.007415451342768288,0.04559523809523809,0.005379271517781076,1.0,0.0,0.325,4.3474363722251854e-08,0.27671641791044777,,,,,,,,,,9.594235055800515e-10,0.07203580397305112,9.594235055800515e-10,Baseline
+187,0,Pref. Opt.,Llama-2-7B,0.0,5,2,1e-05,0.9151450053705693,0.054000000000000006,0.020404330430982054,0.5722097067137865,0.02510219965379874,0.4415312414326696,0.014325977955111896,0.015801948051948053,0.005799916835379152,0.9686818181818182,0.004525620178476433,0.6125356125356126,0.044040543321181944,0.529236215163353,0.03203272049487943,0.43155472876900725,0.019194650359717888,0.08507932384004804,0.00756657825907604,0.936486415854671,0.006163511379818749,0.4628798043496495,0.024953443608491957,0.4206539897593982,0.013253595500654997,0.8649147326882922,0.006087829748181217,0.47078225655905737,0.023999971074593868,0.9474343640993979,0.003872753281966895,,,,0.020908516414312812,0.002904458056869209,0.6053232094145459,0.014664633554087673,0.738499884739785,0.008074424604343905,0.07202182539682539,0.006566093976075974,0.9961580086580086,0.0012136544188184788,0.3,3.925055953685187e-09,0.2599502487562189,,,,,,,,,,2.4125498977354686e-08,0.26420797583813543,2.4125498977354686e-08,Baseline
+188,0,Pref. Opt.,Llama-2-7B,0.0,5,3,1e-05,0.9151450053705693,0.308,0.04522327698370367,0.5789469365563766,0.02623740121312327,0.44668255618802016,0.015339548784365695,0.06826767676767678,0.010558634371925711,0.9326025535490728,0.00418751191685754,0.8219373219373219,0.03425908309815273,0.5251629508206394,0.03312746930049978,0.4297605849433997,0.020305735048434123,0.10801610399137931,0.006721923299455592,0.9292579239033218,0.006255199930134483,0.6913270487742572,0.020248285158319765,0.43338003903901307,0.013865197286935247,0.9017283142412297,0.00596248635971975,0.7091873899679761,0.019462981276456143,0.9288970562926833,0.0034807920279324887,,,,0.05004966970880763,0.011244175501051078,0.583178652704166,0.014989196737112102,0.759909662590831,0.008643109943937185,0.07872367155261892,0.01154137485694821,0.9303075712602089,0.002779957696801995,0.315,1.6808620678088061e-12,0.21218905472636815,,,,,,,,,,3.602472544567432e-09,0.5136001995856703,3.602472544567432e-09,Baseline
+189,0,Pref. Opt.,Llama-2-7B,0.0,5,4,1e-05,0.9151450053705693,0.5706666666666667,0.04896522965255089,0.565499220690223,0.027727109816950018,0.4386541128471928,0.016353903385306057,0.123871441424073,0.011310796141443511,0.958266952360841,0.004767578433928951,0.8433048433048433,0.032704213825379885,0.5197777965403247,0.034058506608159345,0.42520273041212625,0.021280468193949575,0.1082743003506315,0.00623970743761935,0.9304835925042281,0.0062107957109314655,0.7210149719677946,0.02001198828814193,0.4427044068793173,0.014109612484275573,0.917666523612705,0.0055014059662282155,0.758723990592372,0.018657127007372366,0.9333217750631112,0.003682700796757412,,,,0.04729709623144768,0.011820098477290809,0.5709235396390437,0.015130288935448597,0.7551814881408296,0.008669437083403507,0.08114327485380116,0.012161823009855344,0.9857659786910854,0.0024314272365065343,0.325,1.2132904756219738e-15,0.173681592039801,,,,,,,,,,9.594235055800515e-10,0.5625909659356509,9.594235055800515e-10,Baseline
+211,0,Retain Model,Llama-2-7B,0.0,5,-1,1e-05,0.0,0.9205,0.025086667955129253,0.5729132150965522,0.03427057868704381,0.43503571667591834,0.022374244011853502,0.17691183768931448,0.007291097532838744,0.9690777494867144,0.005178207508172694,0.8767806267806268,0.029084330942212015,0.5418455296057031,0.03635483355293133,0.42301799526850226,0.026996039109534263,0.10245494041775863,0.006134363770269276,0.918898716014429,0.006774046669993151,0.9805933234120219,0.005485853467601422,0.4755975328170475,0.014347836485644703,0.9886494358004753,0.000980908811856877,0.9822244335738788,0.005152633769151842,0.9240024171121161,0.0033380822115808375,,,,0.3985728227301341,0.013367695489676859,0.6615929760528791,0.014978314520705974,0.14898405311082313,0.010286737621283272,0.40210312730726927,0.0131396127343718,0.9030987560029818,0.004802385706095919,0.0,1.0,0.0,,,,,,,,,,1.0,0.6158297400808685,1.0,Baseline
+190,0,Grad. Ascent,Llama-2-7B,0.01,5,0,1e-05,0.9473684210526315,0.9329999999999999,0.02378587136868601,0.5925250633981587,0.03313136393208343,0.45866895881856495,0.022665461785013455,0.1802043745727956,0.00732707623791051,0.9728379059461986,0.004856534057645312,0.8917378917378919,0.02791127949453827,0.5674547294800175,0.03554356369979529,0.4478091135657267,0.03121719605340004,0.10098425240047906,0.005960362739992294,0.9128430829735381,0.007119627053711257,0.9706208749496527,0.006304056219499799,0.4774222984068487,0.013770900880081774,0.9786201679749079,0.0016873709794033814,0.9732615550415562,0.006278893381859284,0.9235908002636241,0.003390387242427505,,,,0.8101712253614772,0.018679312173029334,0.5179144772244348,0.014741867415253225,0.9142147623108586,0.007217239323173101,0.8293942977426894,0.01798821504141893,0.9151245107447159,0.004680875814849894,0.405,3.769322074171927e-14,0.19139303482587064,,,,,,,,,,4.734741877153768e-15,0.6336610223711723,4.734741877153768e-15,Baseline
+191,0,Grad. Ascent,Llama-2-7B,0.01,5,1,1e-05,0.9473684210526315,0.938,0.02338867895008829,0.5157342139948607,0.035594422126233254,0.3926203568446402,0.022844657594140456,0.17252240134942567,0.0070345991832812,0.9639644886881197,0.00595477404828427,0.8860398860398859,0.028587489257893277,0.5416839946639962,0.03647590238505109,0.4264966458411362,0.031909980755672994,0.09213112869845316,0.005891717042168777,0.8946734190925885,0.008736833211199687,0.5743844576850133,0.016615610450320232,0.48322775186751843,0.0135537156526867,0.7413954711731404,0.011904485364813503,0.5072584318806999,0.01638736797712621,0.8797476098046074,0.005952749257503866,,,,0.3891593363508902,0.01656645714801791,0.5572400872813026,0.016095738609523274,0.3054323239994752,0.01739810624429241,0.3500566723711785,0.01582105199253559,0.7772350220005068,0.019599411922653947,0.305,9.576382792721885e-08,0.28248756218905474,,,,,,,,,,1.2940880677508261e-08,0.5614232235106072,1.2940880677508261e-08,Baseline
+192,0,Grad. Ascent,Llama-2-7B,0.01,5,2,1e-05,0.9473684210526316,0.8163333333333332,0.036405544033342445,0.4537622302509704,0.03888422123008048,0.3321294542287017,0.02458230040541431,0.08142976944487669,0.00637476836778277,0.7746534800385234,0.017282320783005727,0.8760683760683761,0.029119795619916317,0.5217143616748763,0.03814279687847682,0.40645331250149996,0.035031848053448,0.05429999016703442,0.0030800709712405488,0.8106190171215799,0.008048698280504384,0.3559074880311737,0.010842990865291379,0.44729751166762394,0.015124744996123547,0.11690241572978832,0.005054843895550297,0.17380655321322508,0.006641423483436018,0.5558013922031897,0.015724982642038574,,,,0.2611477059355814,0.013931330080478718,0.6311018235263933,0.017271695841099888,0.024799492901259344,0.0034608877690643484,0.11907708895285211,0.00834522051971428,0.3487415268398668,0.02047449189486617,0.08,0.812879437001558,0.49034825870646764,,,,,,,,,,0.5452713464323318,0.35214436184346476,0.5452713464323318,Baseline
+193,0,Grad. Ascent,Llama-2-7B,0.01,5,3,1e-05,0.9473684210526315,0.008333333333333333,0.005981171354710611,0.586240393128043,0.040028554656667686,0.44921521746343535,0.0352479808935698,0.0011717171717171716,0.0011121513343137909,0.045536275153771635,0.0021601891024510423,0.20185185185185187,0.03370566307841695,0.5996745810765836,0.03830396431497519,0.44644594204684596,0.03740036913574294,0.006553515103391414,0.002525784006792439,0.05281281069372069,0.0023763222299879026,0.0371423353516301,0.0030323499752257766,0.33212135300307355,0.017678535787907654,4.536445021653192e-07,1.929390232514065e-07,0.0077977017103661025,0.00119432842956774,0.037016696127939515,0.001072732594368145,,,,0.012570848268493926,0.002400289873992329,0.6191453354256883,0.019437891193626843,2.2293355989991723e-07,1.1891241252181544e-07,0.0018767366101812189,0.0003725394029903712,0.01986796127175048,0.0009316257876697808,0.295,0.0012872422190080735,0.3687562189054726,,,,,,,,,,4.448673570292354e-08,4.0824991468728534e-06,4.448673570292354e-08,Baseline
+194,0,Grad. Ascent,Llama-2-7B,0.01,5,4,1e-05,0.9473684210526316,0.0,0.0,0.5173993262576859,0.0438132091050021,0.36173334890129255,0.042685097915878364,0.0,0.0,0.013002009739873498,0.0004515061137179245,0.0,0.0,0.5958168968193794,0.03936343654425418,0.43258375565006213,0.04030394390059511,0.0,0.0,0.015241730310709163,0.0006666392474065423,0.002520152505446623,0.0009626307265720538,0.2730750732626796,0.015464941106522228,4.497176550842721e-18,4.184155085356248e-18,0.0001421227113374141,5.319347162349327e-05,0.011536314263832793,0.0002031510004524356,,,,0.0037184162843698267,0.001206216708034702,0.7066584160482915,0.017559412945843858,1.166379924570505e-21,7.493992649615104e-22,0.0004941246650993723,0.00017100842889342726,0.009589911078997222,0.00025181475662140577,0.33,1.2335167589248498e-06,0.30223880597014924,,,,,,,,,,4.869254633582969e-10,0.0,4.869254633582969e-10,Baseline
+170,0,Grad. Diff.,Llama-2-7B,0.01,5,0,1e-05,0.9151450053705693,0.9329999999999999,0.02378587136868601,0.5874238034836436,0.03352375135569227,0.4539579854721211,0.02340217087173837,0.17740739953201254,0.007059005877161073,0.9734813789172929,0.004726584442778651,0.8831908831908832,0.02891603593003461,0.5640663184243199,0.03571971852444417,0.4424416732705837,0.0317898584174373,0.10038494296270732,0.006116595737014666,0.9122151799646189,0.006894236532251265,0.96983841578055,0.006678476645025631,0.47968223050832864,0.01406977340194054,0.9835342877992063,0.0019373291523417057,0.9703292636865084,0.006756381677664956,0.9241442268999053,0.003322106314350041,,,,0.8497164947403658,0.01721410516431764,0.5115331797850922,0.014971808589220714,0.9368338080549877,0.006712313708421566,0.8576277488493678,0.017293115038070305,0.9149675541574319,0.004494616562535683,0.415,2.68549361221259e-14,0.18960199004975126,,,,,,,,,,8.330858010549734e-16,0.6304796812500467,8.330858010549734e-16,Baseline
+171,0,Grad. Diff.,Llama-2-7B,0.01,5,1,1e-05,0.9151450053705693,0.9229999999999999,0.02553883950441608,0.5166283580701875,0.03567677183428179,0.3856739075902042,0.02639207651968379,0.17087880128127808,0.006724748616226653,0.9761070439189357,0.004767924773851973,0.8746438746438747,0.02986595450792445,0.5376730427794998,0.03651540463080886,0.4137046164235319,0.03391496607174791,0.10301848887745502,0.005907437851220956,0.9230355949892957,0.007073858414657302,0.6171630029622562,0.017041320623862617,0.47204449025047107,0.015243874956705601,0.7699543024967371,0.011856312540283555,0.6520166743077185,0.016600761681041593,0.9261942689153981,0.003900590762122225,,,,0.39642783397658976,0.01711442997000426,0.5309601644808373,0.0167477071435362,0.2822650949696279,0.017424720101809597,0.43345947259873824,0.01797143269169379,0.8348928398946078,0.01881788622826199,0.31,3.0050389653580544e-10,0.24318407960199004,,,,,,,,,,6.865581733487208e-09,0.5600557882590118,6.865581733487208e-09,Baseline
+172,0,Grad. Diff.,Llama-2-7B,0.01,5,2,1e-05,0.9151450053705693,0.743,0.04257506739438428,0.5211486378774106,0.03680303235071817,0.3763546568334124,0.025860848770738444,0.13424693009849675,0.009428107580690435,0.9296220725373537,0.014519373431263846,0.844017094017094,0.032427305993378105,0.5301571799248069,0.03823411693286818,0.40949883813002713,0.03524288579350792,0.09482302291667319,0.0057872820559351865,0.9148368112882068,0.007690350454387315,0.35650977493259384,0.012631935309397952,0.4574745948708922,0.01586580607247563,0.3514542187037176,0.012292156224807196,0.3186988837202641,0.01225650978752383,0.8343601573459485,0.007912814128406359,,,,0.2476454028003329,0.01308009056236376,0.5587190661987183,0.01728482594004094,0.04198601275781345,0.0071798127077720994,0.21664903744689323,0.011634685423414694,0.6706129622185059,0.02176288805209075,0.195,0.00027678924136298347,0.3517412935323383,,,,,,,,,,0.0009668691315127085,0.467021512428321,0.0009668691315127085,Baseline
+173,0,Grad. Diff.,Llama-2-7B,0.01,5,3,1e-05,0.9151450053705693,0.48033333333333333,0.049060980401738806,0.5941113318613634,0.03724036020008713,0.41718774025547595,0.02822761448533869,0.05637789125353626,0.007751727642164449,0.5510641572706397,0.03711325215075403,0.8554131054131053,0.030756732005690294,0.5649462235354812,0.03764035998157314,0.425817287326774,0.035550933939448644,0.06767772518831451,0.005460472000221422,0.7674443504937926,0.023276297878161278,0.20687349690788204,0.011829393229021714,0.49740107262413147,0.01681319249201463,0.0998182704250033,0.009219867105464408,0.10461399228383754,0.009476597551743347,0.2519013829059904,0.01509810547049928,,,,0.08359014207631915,0.0068498687351620565,0.5323651323026722,0.0192980320091164,0.0018804820412791849,0.0005141278833844433,0.05143446478616281,0.006626648518444373,0.11674958884342779,0.010428686600522574,0.24,0.7043367047818793,0.48452736318407963,,,,,,,,,,1.8266119303942767e-05,0.31788489192349895,1.8266119303942767e-05,Baseline
+174,0,Grad. Diff.,Llama-2-7B,0.01,5,4,1e-05,0.9151450053705693,0.316,0.044301487258199795,0.619962359998428,0.03934082143531331,0.4489966895973788,0.03191330303939945,0.03740909765135024,0.00803556732811522,0.35805645311319695,0.03395648790704227,0.7792022792022791,0.03625332240438845,0.5881499967242971,0.03816145953438107,0.4440645988346686,0.035913485322087865,0.06134150167914816,0.0053180663303825995,0.6710996795211137,0.03011783839651657,0.10115017695157541,0.006081679162405764,0.5311717261556455,0.020062212191220487,0.0090277310809687,0.002663376644141291,0.07368236069850603,0.0068071410756980295,0.15793767801198053,0.011998838714159792,,,,0.034116932411262624,0.004177714866499876,0.4774132851635447,0.02137342931852981,1.7890158801022174e-05,9.361773674498482e-06,0.024125459609777647,0.005694363090756522,0.08034299289311621,0.010622514099562596,0.3,0.036059809121359955,0.4145273631840796,,,,,,,,,,2.4125498977354686e-08,0.06677597883744442,2.4125498977354686e-08,Baseline
+200,0,KL Min.,Llama-2-7B,0.01,5,0,1e-05,0.8678839957035446,0.9329999999999999,0.02378587136868601,0.5910074162287388,0.033138180499928616,0.4570308648885492,0.02268598700569768,0.18002255639097745,0.007324369791516868,0.9725045726128653,0.0048491364633212715,0.878917378917379,0.02908252649809801,0.5670273118785237,0.03553568712461549,0.4470568263753864,0.031238532977024978,0.09915857409367994,0.006018135823784579,0.9110636274130782,0.007258669691727621,0.9707105108039946,0.0062235451646627995,0.4776343918793863,0.01376691657732712,0.9788741616223882,0.0016990950957590214,0.9737220422667493,0.006081354876672626,0.9239791272023522,0.003372772486128818,,,,0.8064874111318706,0.018796058334244897,0.5177698392221218,0.014780267884736445,0.9143506204447637,0.007264887721850691,0.8260865566285003,0.01806802595331568,0.9149984212795576,0.0047260906258515464,0.405,3.8048842609423915e-14,0.19144278606965173,,,,,,,,,,4.734741877153768e-15,0.6322231899772564,4.734741877153768e-15,Baseline
+201,0,KL Min.,Llama-2-7B,0.01,5,1,1e-05,0.8678839957035446,0.9229999999999999,0.02553883950441608,0.5105777661996006,0.03626781975513106,0.39168486183598106,0.02332527546088273,0.17362247312943907,0.007123247240270153,0.9711576092956838,0.00504947238885096,0.878917378917379,0.02908252649809801,0.5416433598325976,0.036533291053152075,0.424497783729087,0.03221189255410493,0.09994462981690538,0.005896685207324643,0.9177206735578699,0.0075066856465180725,0.6029934138765807,0.017392939338839043,0.4802343021426168,0.013708317216446535,0.7711562460742497,0.01220787807838053,0.5877002612819012,0.017138347653060323,0.9003103062252878,0.0055918074465915865,,,,0.38683401577997,0.01682129563512824,0.5609514985442768,0.016102758444606943,0.3085260541039884,0.017960968253701148,0.3789831883613213,0.017110041651041873,0.7958912887841546,0.019975370252672965,0.3,1.392352220390052e-07,0.285273631840796,,,,,,,,,,2.4125498977354686e-08,0.5634773714164372,2.4125498977354686e-08,Baseline
+202,0,KL Min.,Llama-2-7B,0.01,5,2,1e-05,0.8678839957035446,0.8683333333333333,0.03094608105495642,0.4540266648494174,0.039156552490983806,0.3324999602940135,0.02445007457380271,0.09104113196576138,0.0061630735056413585,0.8243193671458058,0.015199207314434434,0.856125356125356,0.03087679002324079,0.5158509400216378,0.03837723222203142,0.4111121721042936,0.03466429536167537,0.05436684321532199,0.0033204352353553065,0.811918629966512,0.009667500685405459,0.38505714177115014,0.012775950120241727,0.4579126662407413,0.01485329710276641,0.23314815815442644,0.00873762716018992,0.27012591860596447,0.008456961352777641,0.7740651870824939,0.010750033602209264,,,,0.2843598807588266,0.014165103295070137,0.6183329427883892,0.016363611704175652,0.040840640863529086,0.005972834117224615,0.1695322672049081,0.010761311281763143,0.4800695816113781,0.02404054715478152,0.095,0.19587952772142314,0.4472636815920398,,,,,,,,,,0.32811544409418575,0.42917132451396545,0.32811544409418575,Baseline
+203,0,KL Min.,Llama-2-7B,0.01,5,3,1e-05,0.8678839957035446,0.03333333333333333,0.017407765595569787,0.5492191600001007,0.039554826383896534,0.40060631492596577,0.02960921779045337,0.00043230907943684477,0.00022040732401036757,0.05451419161662306,0.0029287503326227084,0.4675213675213675,0.043163669796377584,0.5635520452521665,0.039285003147247644,0.4304441060201373,0.035820477828696316,0.007813357072818022,0.0015333950557498094,0.10035096207627799,0.009975031944997104,0.0651107234797849,0.004743895654306466,0.4682438242698032,0.018792173288577376,0.0003312997083542714,9.36223827678511e-05,0.010737912027427299,0.001177628651783172,0.04108818951413023,0.002300955484046487,,,,0.02982471044559098,0.003791766557327871,0.5739801373760178,0.019833338261956327,8.282259390979095e-05,3.129387500048225e-05,0.00453318274272572,0.00061860601769191,0.0223732905972373,0.0014540787704371012,0.265,0.05571901803165317,0.42199004975124377,,,,,,,,,,1.3921047931216453e-06,0.002925437779133216,1.3921047931216453e-06,Baseline
+204,0,KL Min.,Llama-2-7B,0.01,5,4,1e-05,0.8678839957035446,0.0,0.0,0.5559731084150562,0.04485232283747459,0.4240447305220686,0.04131374612558055,0.0,0.0,0.008496729554591418,0.0003318546383796571,0.008547008547008548,0.008547008547008543,0.6505192937589148,0.037611671783387514,0.46205950560949677,0.03823697093355067,4.940467368213033e-05,4.94046736821303e-05,0.011832147933085175,0.0005585599580346984,0.0034139378347436172,0.0010190250028826035,0.32413716190626146,0.018066540986318175,5.484437260856177e-13,5.381124925733793e-13,0.00028247263449129455,7.385300793755056e-05,0.007974333199994287,0.0001784450186043068,,,,0.005279473282926825,0.0013143259111574887,0.6764715717235333,0.01853092432267541,1.175034691120749e-17,6.728993220807271e-18,0.0007736001144513457,0.00019506467751841702,0.007519532857725062,0.00019338485256453834,0.335,2.727591958251271e-06,0.3087562189054726,,,,,,,,,,2.44374577116149e-10,0.0,2.44374577116149e-10,Baseline
+180,0,Pref. Opt.,Llama-2-7B,0.01,5,0,1e-05,0.9151450053705693,0.9363333333333332,0.0226726819112893,0.63250160039251,0.02870102763060999,0.48653803559462167,0.01841504508316433,0.18834125669651985,0.007967186743351637,0.9732857901165892,0.004909625778698633,0.8746438746438747,0.02924268600158814,0.57266414727897,0.03376827676047963,0.4610953704665203,0.028661400796320697,0.10964738671669312,0.00627522933823352,0.9275837303848706,0.006961242644469332,0.9167264397020538,0.011866028468573679,0.45925821043310067,0.013374034989142827,0.9731992735309412,0.0016827082478127976,0.9737537075838293,0.0070169802206684765,0.9313323217964372,0.0032774996605035626,,,,0.9197109486679469,0.01375634127983136,0.542591361363192,0.014546225543707035,0.9665755943677667,0.0021071542978218973,0.9543885332111318,0.010347062774061265,0.9207817348409733,0.0044579142821053516,0.385,7.07479007508391e-12,0.22044776119402984,,,,,,,,,,1.3261055712130201e-13,0.6401842065170381,1.3261055712130201e-13,Baseline
+181,0,Pref. Opt.,Llama-2-7B,0.01,5,1,1e-05,0.9151450053705693,0.035333333333333335,0.01748380684186089,0.5719617277596022,0.023226343045639787,0.4469203126162885,0.013355173146737578,0.01119047619047619,0.00507186269530058,0.9988888888888888,0.0011111111111111115,0.6324786324786325,0.043559682890086306,0.5287792917056197,0.031104674193368324,0.4395163981667699,0.02097686449128255,0.09364346495098581,0.009166565076973304,0.9563567622334682,0.006166733255569721,0.03350314462129095,0.008186989265911344,0.41443888006482293,0.012222589619469945,0.8100919601321238,0.005179467999086189,0.06422527472527471,0.009818947671044097,0.9966556906253144,0.0013560740581963731,,,,0.010065381033253376,0.0012870526601959987,0.610762175903099,0.013797910058363335,0.7169183700964531,0.006990264785835469,0.041428571428571426,0.0052789418724709615,1.0,0.0,0.345,1.7795961390352562e-06,0.30522388059701494,,,,,,,,,,5.951062636424447e-11,0.12582041664097618,5.951062636424447e-11,Baseline
+182,0,Pref. Opt.,Llama-2-7B,0.01,5,2,1e-05,0.9151450053705693,0.044000000000000004,0.018034012011352032,0.5657979924615009,0.025430529191600437,0.44267866136555384,0.014809091332503455,0.012155122655122655,0.0044816297427680216,0.9769230769230769,0.003921352876750752,0.6296296296296297,0.04363351291406516,0.5193921469718058,0.032195297285833265,0.43411872771112725,0.022429332837192888,0.08363123534371479,0.007123637645497817,0.9387040067974894,0.006186262089118744,0.2721681931187517,0.02399027816013496,0.42009388228743266,0.01303000360553764,0.8682673731492133,0.005648204778203411,0.29857510347956356,0.023369863672347196,0.9684996561418066,0.0031234454687738365,,,,0.015515088289712407,0.0015846814651611701,0.603284460385098,0.014334551761818845,0.7433124428420157,0.007414040124345282,0.05612103174603174,0.005442711162307885,0.9785909090909091,0.002779982121143675,0.32,1.1315677229447544e-06,0.30154228855721393,,,,,,,,,,1.8694649189611297e-09,0.22626094710531636,1.8694649189611297e-09,Baseline
+183,0,Pref. Opt.,Llama-2-7B,0.01,5,3,1e-05,0.9151450053705693,0.324,0.04596412010144801,0.5709697486792342,0.026227951918273908,0.4456931394976701,0.015545660234146219,0.07145967920967922,0.010544560789369278,0.9363181087917927,0.00415655614254506,0.7706552706552707,0.037890131277123595,0.5171574025466787,0.03283922607789303,0.43336956351595946,0.023212337303041355,0.09962378316283318,0.006784356698579559,0.9259397187318575,0.005919180898911343,0.6469567116217113,0.023136443698291886,0.42853874743644194,0.013469898936743302,0.8991469687830842,0.005478206929510033,0.6432234189219332,0.022370166654767036,0.9286550437978428,0.003551237875391303,,,,0.053573943351614924,0.011483570422259297,0.5903420744421378,0.014706602893287405,0.7690024058285008,0.007668201349401644,0.08665050165435828,0.011637839763071292,0.9310429116771683,0.0028553764195649424,0.325,1.814054196884585e-07,0.2872636815920398,,,,,,,,,,9.594235055800515e-10,0.5110256365092916,9.594235055800515e-10,Baseline
+184,0,Pref. Opt.,Llama-2-7B,0.01,5,4,1e-05,0.9151450053705693,0.483,0.048880841853805056,0.5708461882538819,0.026509654647178037,0.4453313821258976,0.015776119967922032,0.10320657936834408,0.011064132499832057,0.9497696189398225,0.004571273115377773,0.8048433048433049,0.035581576089878504,0.5174909878771042,0.03306624902134218,0.4341185574781436,0.023561371139087933,0.10527159688756997,0.006666716908241489,0.9285641538832888,0.00604653134147936,0.6885409446347347,0.02171966122048687,0.43177106169398827,0.013590230853582767,0.9104045661773856,0.005373730650659352,0.703433264154308,0.020877901967121366,0.9346102111787343,0.003508872903052389,,,,0.06157191270509493,0.012440345828952785,0.5845300809512187,0.014825938122426056,0.7790837352682299,0.007772401657066502,0.09579020230500493,0.012969707320406776,0.957788227057558,0.0033168620625591005,0.335,7.113852130985211e-08,0.2802985074626866,,,,,,,,,,2.44374577116149e-10,0.548349302350763,2.44374577116149e-10,Baseline
+210,0,Retain Model,Llama-2-7B,0.01,5,-1,1e-05,0.0,0.9205,0.025086667955129253,0.5744998213253056,0.03451290382456747,0.437720092187052,0.022459274020778394,0.17376498003200788,0.007230587835081752,0.9675397550107209,0.005571685794012701,0.8846153846153846,0.02814097500969468,0.5412220740042258,0.036419287094437335,0.4224764930645468,0.02698754891907263,0.10266562621482804,0.005822990325217776,0.9172968708805942,0.00672453135500215,0.9813574811531377,0.00544891128489239,0.4797529373527588,0.014340055451626936,0.9886701137754393,0.0010148843418614384,0.9828811468322961,0.005204556441877092,0.9242021237952271,0.003326192690789866,,,,0.3928703375582458,0.013401275383845564,0.6619565180041925,0.01527184357495311,0.14896743498501294,0.010450251021317839,0.39924306642773827,0.01307849551579204,0.9026594195885775,0.004694304724718772,0.0,1.0,0.0,,,,,,,,,,1.0,0.617642376161701,1.0,Baseline

versions/modify.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Loop through each CSV file in the current directory
+for csv_file in *.csv; do
+    # Check if the file is a regular file
+    if [ -f "$csv_file" ]; then
+        echo "Processing $csv_file..."
+
+        # Temporary file
+        temp_file=$(mktemp)
+
+        # Check if the file has a header
+        if head -1 "$csv_file" | grep -q "Submitted By"; then
+            echo "The 'Submitted By' column already exists in $csv_file."
+            continue
+        fi
+
+        # Add 'Submitted By' column header and 'Baseline' entry for each row
+        awk -v OFS="," 'NR==1 {print $0, "Submitted By"} NR>1 {print $0, "Baseline"}' "$csv_file" > "$temp_file"
+
+        # Move the temporary file to original file
+        mv "$temp_file" "$csv_file"
+
+        echo "Column 'Submitted By' added successfully with 'Baseline' entry in each row for $csv_file."
+    fi
+done
+
+echo "All CSV files processed."

versions/phi-10p/phi-10p.csv

@@ -0,0 +1,45 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+104,0,Grad. Ascent,Phi,0.01,10,0,1e-05,0.8888888888888888,0.38733333333333336,0.04848034069953959,0.46262685920343233,0.032022274818019815,0.37727909801521875,0.024351149126135403,0.006216376446533307,0.0008325163165586139,0.33363565117459865,0.002581931553834998,0.7813390313390313,0.036329601764307544,0.49764006590077964,0.029303189670398327,0.42394593734073427,0.024844152263675656,0.008698525332898381,0.000609437555348295,0.32001749255522816,0.0027811474692140668,0.8293988903501516,0.013375125953831815,0.4675757684708973,0.014892585442756652,0.8856740242073007,0.0045197665070554624,0.13599701940092346,0.0034752642065795894,0.4204511387351163,0.006238060685493297,,,,0.7739077047207454,0.014218989234627513,0.4962392731653131,0.013514095540609302,0.8513315066446633,0.006550243701946486,0.1505747906404153,0.00416185324293904,0.43851280496589123,0.005785546183555594,0.3433333333333333,1.560351168372984e-15,0.23448504983388704,,,,,,,,,,4.499933086722873e-16,0.5138312128126559,4.499933086722873e-16,Baseline
+105,0,Grad. Ascent,Phi,0.01,10,1,1e-05,0.8888888888888888,0.050333333333333334,0.01919534525506847,0.40805294450029195,0.033176226664097086,0.33773306442955425,0.025145873823857533,0.0008310899085380647,0.0003160929497113445,0.38256543081324046,0.006163353383665441,0.5066951566951566,0.043822581683795284,0.4575550487490544,0.029849815468665587,0.41641462641092025,0.027915693718454464,0.005409955411943765,0.0005465415547370757,0.3210891134762343,0.003392912980891136,0.3753649794084481,0.008758192077520759,0.4079706719520729,0.01604554754578335,0.20340229312804473,0.009048480887113085,0.05748510143554693,0.0011990444908374707,0.40899719051172007,0.005581217341813106,,,,0.3685740816396321,0.008724358383170072,0.5697747610953693,0.013634570700029058,0.15178419933425014,0.008786395278742023,0.06789342072111246,0.0014919333319953292,0.4424170946570548,0.005994998501491427,0.18333333333333332,1.3019995884475283e-05,0.3547951273532669,,,,,,,,,,8.024952455325493e-05,0.21494446099725653,8.024952455325493e-05,Baseline
+106,0,Grad. Ascent,Phi,0.01,10,2,1e-05,0.888888888888889,0.0,0.0,0.39158088345838826,0.03800380120509807,0.3075735683060553,0.029857819211246285,0.0,0.0,0.02959125094122237,0.00048306317239815895,0.0,0.0,0.2854042734525795,0.03082270522742792,0.3360544677760921,0.03222595389640453,0.0,0.0,0.027940060567179213,0.0003859521499626529,0.007807156410915588,0.0010601901322647809,0.17270937948507792,0.012886329657118577,1.8382741673714414e-21,3.9788466594068538e-22,0.0013606045995862963,0.00016400379703431758,0.029495057002204183,0.00027231076232216384,,,,0.006670329759095243,0.0009573188423209912,0.7163483706949536,0.011714719455172608,1.0776189832024882e-21,1.4469548801109098e-22,0.0014906642925606281,0.00020001780970441,0.030475322446632928,0.00028735312281396273,0.23666666666666666,4.848462497562527e-06,0.3477519379844961,,,,,,,,,,8.835663084782468e-08,0.0,8.835663084782468e-08,Baseline
+107,0,Grad. Ascent,Phi,0.01,10,3,1e-05,0.8888888888888888,0.0,0.0,0.4129156176796982,0.0411221816671796,0.2931598040099206,0.03438947237267602,0.0,0.0,0.025401168476535216,0.00041550134386492425,0.0,0.0,0.34299333163574197,0.033866094896031075,0.3493050898894228,0.034138717826545706,0.0,0.0,0.025816242765395313,0.00036380379433475455,0.007550746154505331,0.0010349914605595788,0.16370338977878376,0.014435452971855237,4.9517116102927254e-33,1.2883600309977162e-33,0.0011470510031240104,0.00013985878728187777,0.027087213146201026,0.00022403677230231216,,,,0.006485144573910058,0.0009435035460830455,0.6911288732463645,0.013104429787068127,2.0499950950854926e-33,5.448030113526146e-34,0.0012641290999910917,0.00017210636833645877,0.027795582898943724,0.00025674750627736854,0.2866666666666667,6.439154694790821e-07,0.33421926910299005,,,,,,,,,,2.8915775251270757e-11,0.0,2.8915775251270757e-11,Baseline
+108,0,Grad. Ascent,Phi,0.01,10,4,1e-05,0.8888888888888888,0.0,0.0,0.42272137701102525,0.04198985879761028,0.29214401137254276,0.03439819729145634,0.0,0.0,0.023413290513385775,0.0004008956370943507,0.0,0.0,0.3514976578827366,0.03438030776644592,0.3517272798800346,0.03451353564607756,0.0,0.0,0.024839786704193487,0.00037164006895920734,0.007550746154505331,0.0010349914605595788,0.16875376753134938,0.014809581127576407,6.128942235346742e-37,1.7602274676345458e-37,0.0011137794051070903,0.00013579294552203738,0.025437145867752035,0.00022487348273128978,,,,0.00660008710264569,0.0009478526502965091,0.6903274669246469,0.013843159240812183,2.6949097075668354e-37,8.985990808942673e-38,0.0012497410581228368,0.00016856372800777225,0.02509197975700741,0.0003157540558186838,0.31333333333333335,1.0799718480933462e-06,0.33758582502768547,,,,,,,,,,2.0608068972918464e-13,0.0,2.0608068972918464e-13,Baseline
+84,0,Grad. Diff.,Phi,0.01,10,0,1e-05,0.8208616780045351,0.38733333333333336,0.04848034069953959,0.46763445442266877,0.032086944521973707,0.37983783104513946,0.024551987007587654,0.0062214564383918595,0.0008320447075543163,0.33967725891835315,0.0036467775257637324,0.794159544159544,0.035702313401374315,0.5004589107540602,0.02922319247158341,0.4238985762130126,0.024852575894821077,0.008833055175520275,0.0006069072846750914,0.3209691273481886,0.0029922111857795773,0.8512018888944253,0.013617520716051027,0.47289773805319774,0.014868782794743247,0.9022726216633026,0.004059745092963582,0.13979093243781576,0.0036225120116489726,0.4209939501709632,0.006246072991701256,,,,0.8068307904481231,0.013967347966220936,0.4912515257885599,0.013589591988715675,0.8747724664497272,0.0057496242088185085,0.15874476881433583,0.004369379838342959,0.44962312237478813,0.006518956081873533,0.36,3.3244427687029583e-16,0.22819490586932448,,,,,,,,,,1.150018983956209e-17,0.5182230783786858,1.150018983956209e-17,Baseline
+85,0,Grad. Diff.,Phi,0.01,10,1,1e-05,0.8208616780045351,0.26733333333333337,0.043178901195115055,0.4708035736212265,0.03227237204878717,0.3778543917353441,0.025604040656350004,0.004188064407966967,0.000695019857980829,0.34197609229103,0.004346917977793288,0.671937321937322,0.040765954548627116,0.48772073795171245,0.02944815890148784,0.42496179063474276,0.026236346711911676,0.007254724298555321,0.000572458500660811,0.32733186105132894,0.003783748731779358,0.5800804822838824,0.013248426383885528,0.4560462773151391,0.015229796830433847,0.6964530245978232,0.009380494396936676,0.0919533604161501,0.00258634036676916,0.41580093804498214,0.005216825541800492,,,,0.46834452300903495,0.01081264023627754,0.512469669007763,0.013630596946085581,0.41841045483635875,0.013129476230542697,0.08727695613153873,0.0021786302405923615,0.44446069709361763,0.005680044628805669,0.2833333333333333,1.2756156971881535e-12,0.2636323366555925,,,,,,,,,,5.191390347437448e-11,0.4555706116355508,5.191390347437448e-11,Baseline
+86,0,Grad. Diff.,Phi,0.01,10,2,1e-05,0.8208616780045351,0.1895,0.038614221224933194,0.4786367966447317,0.03242327265239884,0.37907945579618435,0.026582851236194017,0.0029103807314015107,0.0006066207759563108,0.4133417677176665,0.008601314832276769,0.5935897435897435,0.043360004371740875,0.47753891647194047,0.029658474826921136,0.4232705690329648,0.027610508124742066,0.006268795915061675,0.0005520153472636709,0.3580717789942354,0.004939226119811255,0.4958000890884974,0.012288113694150487,0.4472732125103214,0.015632229754474054,0.5312651643551275,0.011947699596160564,0.07773724830857956,0.002037444089024772,0.46385600611004035,0.006516757143451464,,,,0.36838422577302615,0.008818899867762822,0.5535800898772767,0.013298086551446792,0.11345422380320618,0.00795545892847003,0.07061406004267429,0.001961289993992192,0.49294025218373916,0.006683942217872004,0.21,3.2806140673387666e-07,0.32992248062015506,,,,,,,,,,3.3273813118952403e-06,0.4039391680154967,3.3273813118952403e-06,Baseline
+87,0,Grad. Diff.,Phi,0.01,10,3,1e-05,0.8208616780045351,0.09733333333333334,0.0288565475533775,0.49321620323279936,0.03322858999189783,0.3791272034933522,0.02935485334563536,0.0015375094338432344,0.0004598241720230491,0.4249761946479158,0.011544925797175126,0.4853276353276353,0.04420116625448147,0.46752091517891275,0.030814503271693928,0.41441848746449306,0.028546572985064012,0.005429759327983811,0.0005775995481994158,0.3791365272525054,0.009337505761420018,0.3967442069196385,0.010983910952358254,0.4502406648146126,0.016485460492941202,0.3697677614769308,0.011846399079176693,0.062395476391998154,0.0015834503349666136,0.4808229704484246,0.007391773643496799,,,,0.2729516026853662,0.007406683059459851,0.5780388987583464,0.013627516496579957,0.024093259145547375,0.003073711444596511,0.055381529496401995,0.0020733294223912986,0.4992073897012985,0.00780970206944374,0.12333333333333334,0.0025239944311002244,0.3994019933554817,,,,,,,,,,0.020771538379556198,0.3103716513302391,0.020771538379556198,Baseline
+88,0,Grad. Diff.,Phi,0.01,10,4,1e-05,0.8208616780045351,0.09533333333333335,0.02796510572208047,0.4963758383771545,0.034635359808800184,0.38086837687320546,0.031029522946684318,0.001462990304705662,0.0004327746638203656,0.41956301214417935,0.010552117384408847,0.42763532763532763,0.043866509491761074,0.46927195705813685,0.031233518915843424,0.4135819953761371,0.028776294968055657,0.004668542413603897,0.0005472993613465601,0.3731232978428005,0.007377753087154706,0.4048896401235171,0.010767413455804484,0.46830073949996176,0.016727806921783897,0.3802592738348547,0.01183638810785765,0.06350044482587289,0.001613143504834094,0.4818233395827019,0.0071431205953175715,,,,0.25060211740619376,0.006340483445416944,0.5634268277938057,0.014319464493400123,0.011870034047826564,0.002066130578169988,0.05196181616165727,0.0018597898147256979,0.46861735966198126,0.0076666225822631,0.12,0.008048165118045552,0.41173864894795126,,,,,,,,,,0.026515154829730034,0.30767161649743757,0.026515154829730034,Baseline
+114,0,KL Min.,Phi,0.01,10,0,1e-05,0.7210884353741497,0.39399999999999996,0.048862205757775105,0.4630496447337228,0.03213711883559407,0.3777957209815736,0.024383258340669955,0.006282212746001334,0.0008344740976598885,0.33314233201004045,0.002534777962803933,0.7745014245014245,0.037116781834202695,0.4976417096399765,0.029269135286987727,0.42489118439963386,0.024852895528409916,0.008452600336644866,0.0005690156726456839,0.32216641910797494,0.004255364084461845,0.833095776501914,0.013261564291550761,0.4675369302546492,0.014914621323978418,0.8861953804998381,0.004519310572557787,0.13660658417848504,0.0034688456079135215,0.4129806816057001,0.005751563291042689,,,,0.7753096760661167,0.014220579199807251,0.4956776948018333,0.013517215581609338,0.8522478486583368,0.006514076572564693,0.15068938130380238,0.0041471337685869735,0.4384234943484155,0.0058262068935888795,0.3433333333333333,1.3061097204676333e-15,0.23375415282392026,,,,,,,,,,4.499933086722873e-16,0.5152749051510663,4.499933086722873e-16,Baseline
+115,0,KL Min.,Phi,0.01,10,1,1e-05,0.7210884353741497,0.07366666666666666,0.02350487687295065,0.4155601506086474,0.03312215149358541,0.3451285349316393,0.025347802689850867,0.001105369704523463,0.00035145420373549556,0.36671817656130534,0.005241303359357983,0.5309116809116808,0.043521804499529665,0.46316761353905384,0.029743190085519216,0.4178615443633406,0.027896169993041186,0.005761567542448533,0.000553544301479423,0.32393449018906956,0.00343416101179692,0.38418055926336253,0.009276913165971314,0.4145619402598622,0.015829556085932135,0.2379925419887717,0.009649080449063548,0.058454221635296906,0.0012179762311111495,0.4046349328059282,0.005366597822718401,,,,0.37820502225943786,0.0088093550722838,0.5628668604997924,0.013723868852758577,0.1708351194516126,0.00929994070429681,0.06952707800291084,0.0015152492817266084,0.437194231609259,0.005937133700409873,0.19666666666666666,3.1363745056623183e-06,0.34473975636766335,,,,,,,,,,1.727127220822253e-05,0.26063388064586984,1.727127220822253e-05,Baseline
+116,0,KL Min.,Phi,0.01,10,2,1e-05,0.7210884353741497,0.0,0.0,0.38154814653360697,0.0374856018543314,0.3041523225569939,0.029831134719355612,0.0,0.0,0.02562751725952336,0.0003180164076825011,0.0,0.0,0.29191706057482825,0.0307483046993774,0.33923802462678515,0.03297242946188308,0.0,0.0,0.025664640311617536,0.0006693827841054337,0.00770947631323549,0.0010432572338827095,0.17719536249578013,0.012954430634114807,1.8598462952539698e-20,4.590248108523514e-21,0.0013759516290399157,0.00016586953784826402,0.026388300071568425,0.0007807953503177587,,,,0.006485144573910058,0.0009435035460830455,0.7158215846454975,0.011735982867188155,1.012939759217286e-20,1.401166930596452e-21,0.001483468173788356,0.0002013855984698145,0.02676195084147439,0.00019463243188641783,0.23666666666666666,8.012881599741023e-06,0.3512956810631229,,,,,,,,,,8.835663084782468e-08,0.0,8.835663084782468e-08,Baseline
+117,0,KL Min.,Phi,0.01,10,3,1e-05,0.7210884353741497,0.0,0.0,0.40420326107239757,0.041324569269453514,0.2978631451237876,0.0342175538827675,0.0,0.0,0.02618922243692677,0.00032609582066435054,0.0,0.0,0.3413459311862769,0.034245885286102805,0.35318441660494926,0.034525716502273505,0.0,0.0,0.025733233077865847,0.0003419849702186944,0.007550746154505331,0.0010349914605595788,0.15758162250506372,0.014201493539273491,4.604809671424778e-32,1.558734512438092e-32,0.0011965608203791966,0.00014582392025955042,0.027101196288802486,0.00018359316119752285,,,,0.006485144573910058,0.0009435035460830455,0.6937888757878856,0.013207218456694748,1.3791099959872322e-32,3.870876628717119e-33,0.0013204569705441997,0.00017934853597741893,0.027762149140351095,0.00022997203068911593,0.29333333333333333,3.303772651314238e-07,0.3299667774086379,,,,,,,,,,8.777796874606567e-12,0.0,8.777796874606567e-12,Baseline
+118,0,KL Min.,Phi,0.01,10,4,1e-05,0.7210884353741497,0.0,0.0,0.42170661496212086,0.041831969404182504,0.2959215955627443,0.03444527169986282,0.0,0.0,0.027220825735514254,0.0004152865922803097,0.0,0.0,0.349333945584688,0.034616834165588604,0.3587513553879525,0.035146228701785534,0.0,0.0,0.026595004702349337,0.0003716612851068134,0.007550746154505331,0.0010349914605595788,0.16628244713990478,0.014684766917829667,6.053119977894884e-36,2.3721767213170554e-36,0.0011607057995181547,0.00014156054942761332,0.027673896942593097,0.00022694332280337194,,,,0.006485144573910058,0.0009435035460830455,0.6865021592496651,0.013688811798973874,1.536643954075805e-36,5.506393325624413e-37,0.00128028270007138,0.00017393647392277578,0.028988759550723546,0.00027126744537475466,0.30666666666666664,3.025304134640825e-07,0.3294130675526024,,,,,,,,,,7.411265830324788e-13,0.0,7.411265830324788e-13,Baseline
+94,0,Pref. Opt.,Phi,0.01,10,0,1e-05,0.8208616780045351,0.37733333333333335,0.04823455597984076,0.4633567314257481,0.031895566069685424,0.3798519757802948,0.023935430681608423,0.006099201999879242,0.0008343713769010665,0.33312409093633194,0.0032418001511948557,0.7663817663817662,0.03796208513848588,0.5117682133403173,0.027515698378866003,0.4215456639612797,0.023738589524666693,0.008528132029654637,0.0006156955383397769,0.31744980507507875,0.0034052276864741625,0.7897261838427109,0.01563802729883126,0.45837838698468575,0.014526998730112898,0.876547121903362,0.004085892585188771,0.1283068888469919,0.0037150444828089667,0.4219954255233259,0.006351100216257766,,,,0.7546843827842254,0.01634024528593348,0.5071083314830976,0.013517231647519482,0.8757619468497647,0.004291819666333544,0.14801686170708653,0.0048171458204273486,0.4391319201856375,0.006398514647976133,0.31666666666666665,2.984159766716305e-14,0.24691029900332226,,,,,,,,,,1.0747499261825832e-13,0.5096056284904323,1.0747499261825832e-13,Baseline
+95,0,Pref. Opt.,Phi,0.01,10,1,1e-05,0.8208616780045351,0.154,0.03499167770224098,0.41330351236861745,0.030850721589609102,0.35248413855146393,0.02190020702275666,0.002682525631143193,0.0006397126630936975,0.4125072026824579,0.014091135134307015,0.6652421652421652,0.042799260784633864,0.4992958656168068,0.02721888818980077,0.41163504838706727,0.02154968704870177,0.007383153510713886,0.0006438616432829957,0.3658834428768147,0.010231831520669461,0.42245825150406946,0.019488912310364474,0.4419272923709567,0.013924050722171908,0.768253946790184,0.005857639027553714,0.06771987432311334,0.003450680497431019,0.516505678105399,0.00847285607791128,,,,0.22019244482607556,0.008334882547690483,0.5309133421155721,0.013175896194486284,0.7215069924162151,0.005939747765760253,0.04480640078827815,0.0020663336569433245,0.540605987699674,0.008148375029006512,0.2866666666666667,6.8771309671958784e-12,0.2715171650055371,,,,,,,,,,2.8915775251270757e-11,0.38108177504512797,2.8915775251270757e-11,Baseline
+96,0,Pref. Opt.,Phi,0.01,10,2,1e-05,0.8208616780045351,0.26233333333333336,0.043390676898970394,0.4277477813762279,0.031236946351923783,0.3610252010635058,0.02240931780501696,0.004410726182793019,0.0007719185095288448,0.3173188556660405,0.0034406276322295495,0.7378917378917379,0.039971471109039154,0.4929009396784097,0.02813357387533797,0.415959444537795,0.021781931300940752,0.008219607083925379,0.0006348949913872733,0.31005949715019726,0.004802090713059285,0.638810025773943,0.01975709794852587,0.45794313618269183,0.014086515007489855,0.8285163632527844,0.005102581074406367,0.10282600804857303,0.003874041371792119,0.3989700564755297,0.006337097375749005,,,,0.2789320557692803,0.014882761547760234,0.515232306613032,0.013277535065660815,0.7816773556300477,0.00541096972794901,0.05604794177960618,0.003269608440887674,0.372958495372734,0.006923965860153254,0.30666666666666664,1.0063266916163322e-13,0.25220376522702104,,,,,,,,,,7.411265830324788e-13,0.457693302147046,7.411265830324788e-13,Baseline
+97,0,Pref. Opt.,Phi,0.01,10,3,1e-05,0.8208616780045351,0.26233333333333336,0.043390676898970394,0.43219749452172285,0.031272015948036816,0.3633207821420204,0.022588833207167095,0.004563926174495639,0.0008042619722793832,0.321007172838772,0.003044589417687344,0.725071225071225,0.04037902502295831,0.49301821804149476,0.028302286688641962,0.41630717201079037,0.02224397691883354,0.008071939289661037,0.0006300502304919065,0.3172128600143534,0.00457817910188285,0.6623012752874878,0.019921995831522796,0.463795670397099,0.014239312232909173,0.8498283901763967,0.004726918156070472,0.1057432097529181,0.003842153881886251,0.38909406036551225,0.005652573752442972,,,,0.26418368759954414,0.013953248977438203,0.5061117310098682,0.0133431234128358,0.7900949128168973,0.005431819020997476,0.052060315044795843,0.002839327439682139,0.35706992312900676,0.0064992315823277115,0.31,9.635514732762048e-15,0.24208194905869324,,,,,,,,,,3.922502017067572e-13,0.4608206054374006,3.922502017067572e-13,Baseline
+98,0,Pref. Opt.,Phi,0.01,10,4,1e-05,0.8208616780045351,0.26233333333333336,0.043390676898970394,0.43165712120425864,0.03134336095034514,0.36286357801559377,0.022665043771968443,0.004567036572884703,0.0008066638333821726,0.3243596281097105,0.0033138196762159617,0.707977207977208,0.04116164037882798,0.4904415874697781,0.028335928612244935,0.4147520074511906,0.022331424884781834,0.007835425980165777,0.0006347918661963127,0.32330137793897257,0.0041925052876779535,0.6811029982146692,0.019661860317818046,0.465468840851521,0.014237484835500795,0.8565183317546117,0.004568546698867835,0.10914225775713474,0.003896249698376956,0.3870951986600262,0.005390354440795478,,,,0.2506558185868464,0.013390302432822288,0.5050272540974207,0.013363368007215626,0.7901141128010813,0.005431792941650961,0.04924391651848003,0.0026984790275368974,0.3593234280187344,0.005993863525875824,0.31666666666666665,7.491506747594435e-15,0.2410188261351052,,,,,,,,,,1.0747499261825832e-13,0.46080394367478766,1.0747499261825832e-13,Baseline
+124,0,Retain Model,Phi,0.01,10,-1,2e-05,0.0,0.44233333333333336,0.04919065499381502,0.45548746953411495,0.03282125068696775,0.3765961037546308,0.01967854277413743,0.007101746086927905,0.0008489929496740386,0.35242810046475526,0.0061130904487856404,0.7585470085470085,0.038671773713589416,0.5056259057182113,0.029292189971102132,0.4171068998069627,0.018053646341038714,0.008409426191133065,0.0006210314938087412,0.33926658175661484,0.005327860757806458,0.8597069603506552,0.012946990434007403,0.4779785839839506,0.01441199280630298,0.903414796164556,0.004135591404617512,0.1438288427157006,0.003777349529652374,0.4412394141699741,0.007895720453214475,,,,0.4285571553058628,0.008995299662649859,0.6356051211462183,0.012795643890715799,0.13677697900263167,0.0073686247339445066,0.0796810266763256,0.0017165775494871107,0.4627662259818847,0.007559476479293571,0.0,1.0,0.0,,,,,,,,,,1.0,0.524251920557396,1.0,Baseline
+52,0,Retain Model,Phi,0.01,10,-1,2e-05,0.0,0.44233333333333336,0.04919065499381502,0.45548746953411495,0.03282125068696775,0.3765961037546308,0.01967854277413743,0.007101746086927905,0.0008489929496740386,0.35242810046475526,0.0061130904487856404,0.7585470085470085,0.038671773713589416,0.5056259057182113,0.029292189971102132,0.4171068998069627,0.018053646341038714,0.008409426191133065,0.0006210314938087412,0.33926658175661484,0.005327860757806458,0.8597069603506552,0.012946990434007403,0.4779785839839506,0.01441199280630298,0.903414796164556,0.004135591404617512,0.1438288427157006,0.003777349529652374,0.4412394141699741,0.007895720453214475,,,,0.4285571553058628,0.008995299662649859,0.6356051211462183,0.012795643890715799,0.13677697900263167,0.0073686247339445066,0.0796810266763256,0.0017165775494871107,0.4627662259818847,0.007559476479293571,0.0,1.0,0.0,,,,,,,,,,1.0,0.524251920557396,1.0,Baseline
+126,0,Finetune Model,Phi,0.0,0,-1,2e-05,0.0,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.8752167884046598,0.011809089858149302,0.4882534392910444,0.013534006024118014,0.9086918278343834,0.003939262001123021,0.17179131104641562,0.004405566476237556,0.4299893732259693,0.005748680600806638,0.3566666666666667,4.665567748131161e-44,0.0363233665559247,,,,,,,,,,2.4311282147882553e-17,0.515509929080137,2.4311282147882553e-17,Baseline
+109,0,Grad. Ascent,Phi,0.0,10,0,1e-05,0.8888888888888888,0.4073333333333333,0.04890628707870714,0.4406356242224836,0.03210465763045946,0.3683157496287917,0.018886394821732384,0.006476424119882822,0.0008317040505304013,0.3281049493378071,0.00270155455346914,0.7745014245014245,0.037116781834202695,0.4943866037355289,0.029147973767468638,0.4089421237929396,0.018279191875884542,0.008424521654309295,0.0005611561370451387,0.3168406471723739,0.0032912261781003235,0.8293331366032762,0.013525546360893025,0.47124110107624007,0.01475459702588147,0.8860145649500396,0.004550883049867538,0.13578979836134497,0.003489315454593434,0.40927888095291337,0.00602288582473178,,,,0.7745161978784337,0.014187720587384225,0.49534965382860546,0.013520690219857377,0.851142888237336,0.0065772894616179575,0.1495379327598032,0.004073439298777187,0.42869549886833974,0.005696280124063535,0.3466666666666667,1.1902067485709357e-42,0.04409745293466224,,,,,,,,,,2.1942743021891237e-16,0.5097650710521427,2.1942743021891237e-16,Baseline
+110,0,Grad. Ascent,Phi,0.0,10,1,1e-05,0.8888888888888888,0.026999999999999996,0.012514368845827887,0.3787143458174361,0.032383266620030986,0.3285012734983283,0.018715599264884003,0.00048116390140347796,0.00020275372126647503,0.3873666230932246,0.00739301124598016,0.4582621082621083,0.04323131791377336,0.4461277609377497,0.029581617716401953,0.37742364797320005,0.01741689373131969,0.004846520129004641,0.0004991707096444833,0.3269568935592411,0.00529632256531904,0.3434572265997768,0.008519663779399804,0.39415218335462315,0.0161702504941777,0.1332832081459894,0.007439459716360153,0.05244859939126453,0.0010695279205143774,0.4044743200651449,0.0056027076540315895,,,,0.34396669040323125,0.008341377453847346,0.5738379362702275,0.013596302802225898,0.10191279829692185,0.007048959803597189,0.062478826837356775,0.0012873148879500539,0.4384827933379019,0.006286328256681316,0.17,6.420260655107087e-06,0.34972314507198227,,,,,,,,,,0.00033397069759294196,0.14343459129668779,0.00033397069759294196,Baseline
+111,0,Grad. Ascent,Phi,0.0,10,2,1e-05,0.888888888888889,0.0,0.0,0.3861709819636343,0.04079175692534878,0.2927623483867784,0.030142437208385237,0.0,0.0,0.02792804434404739,0.00043609910943842876,0.0,0.0,0.3086000631073772,0.03279741647499922,0.23566999813714237,0.017652880701749328,0.0,0.0,0.027518631190947575,0.0003932350785266071,0.007550746154505331,0.0010349914605595788,0.15638599425486044,0.013326466578662037,1.20699547623219e-26,3.349158753988646e-27,0.0012130687250796128,0.00014796223738014504,0.02878088425650157,0.0002499294619913251,,,,0.006485144573910058,0.0009435035460830455,0.7141512266191344,0.012899828001606358,5.961178331925751e-27,1.0676843705845748e-27,0.0013202579129019303,0.0001796465494887684,0.0286744275144207,0.0002774535149486274,0.26,2.2802826393830683e-06,0.34256921373200444,,,,,,,,,,2.550959337009968e-09,0.0,2.550959337009968e-09,Baseline
+112,0,Grad. Ascent,Phi,0.0,10,3,1e-05,0.8888888888888888,0.0,0.0,0.43768324488741156,0.04286377622761245,0.27893965536690163,0.032246404177528686,0.0,0.0,0.011330985493835564,1.958102121849729e-05,0.0,0.0,0.35469792312892506,0.0353245275189434,0.2538575890783119,0.01912899300259437,0.0,0.0,0.011379685390984825,3.0772137439828354e-05,0.007550746154505331,0.0010349914605595788,0.16579268503752392,0.01493512828320238,1.3953754830761458e-39,4.345321169828471e-40,0.001087085413713033,0.00013248729005034838,0.011481522077037848,1.091040809265311e-05,,,,0.006485144573910058,0.0009435035460830455,0.6899198206632992,0.014186401488148285,7.251360296796451e-40,2.6427886941909497e-40,0.0012010521474301438,0.0001646876652112906,0.011994085443504188,3.169031440214118e-05,0.30666666666666664,9.239368668537131e-06,0.35231450719822816,,,,,,,,,,7.411265830324788e-13,0.0,7.411265830324788e-13,Baseline
+113,0,Grad. Ascent,Phi,0.0,10,4,1e-05,0.8888888888888888,0.0,0.0,0.43865777180439325,0.044329451818748394,0.2794223399907739,0.03330419952607949,0.0,0.0,0.011330985493835564,1.958102121849729e-05,0.0,0.0,0.3656102077269917,0.03668139895918216,0.25480435990722056,0.019639223434811746,0.0,0.0,0.011379685390984825,3.0772137439828354e-05,0.007550746154505331,0.0010349914605595788,0.16850754784082062,0.015408343280255494,1.2665311763692826e-42,4.457082789857745e-43,0.001087085413713033,0.00013248729005034838,0.011481522077037848,1.091040809265311e-05,,,,0.006485144573910058,0.0009435035460830455,0.6760347428655031,0.014446783080401208,6.3879918785462964e-43,2.81229533821925e-43,0.0012010521474301438,0.0001646876652112906,0.011994085443504188,3.169031440214118e-05,0.30666666666666664,7.074335639783706e-06,0.35040974529346625,,,,,,,,,,7.411265830324788e-13,0.0,7.411265830324788e-13,Baseline
+89,0,Grad. Diff.,Phi,0.0,10,0,1e-05,0.8208616780045351,0.38733333333333336,0.04848034069953959,0.445489286930888,0.03229003674923452,0.37030142536660876,0.01888994796965704,0.0061201590578911095,0.000819574741197496,0.33036224192508873,0.003606786695587268,0.777065527065527,0.036862340874226185,0.49531109821206637,0.029118029847928672,0.408587487805685,0.01831624708872293,0.008608980022137672,0.0006026885472454966,0.31927911942242043,0.0040052762824905825,0.8530475707793382,0.013514286562889284,0.47616832991712404,0.014724263389256042,0.9020615519919413,0.004077912862438643,0.13998843202607217,0.003577178138639071,0.40759786609109755,0.0058559235535909455,,,,0.8089788999178928,0.013968280336619586,0.49048439694787827,0.013574288877985733,0.8746356061656911,0.005757043972610084,0.15701386033285988,0.004224819428420731,0.4299783888180241,0.005867888812475599,0.36,1.180529730554766e-43,0.038538205980066444,,,,,,,,,,1.150018983956209e-17,0.5095916359417909,1.150018983956209e-17,Baseline
+90,0,Grad. Diff.,Phi,0.0,10,1,1e-05,0.8208616780045351,0.23366666666666666,0.04134996135612024,0.448711121936769,0.03251591687870528,0.37259156149460004,0.019343586979528517,0.0036174684295270133,0.0006456670789976135,0.35084684639490876,0.004892543566542066,0.6501424501424501,0.04247327893615494,0.4753932430999148,0.029982393492054334,0.4005848724076532,0.018609746294909576,0.0068690105123928825,0.0005716981944819044,0.3303812523029244,0.003546827046213594,0.5628412560299246,0.013110945676236323,0.4580760565067743,0.015229737572763568,0.681057023106619,0.01024070126834248,0.0886238351318877,0.002426277583440824,0.42055467416083575,0.004994579398505561,,,,0.46203609392415806,0.01091867307056635,0.5114268242745105,0.013696751629091402,0.39126369530708427,0.013219304645635324,0.08462003634647551,0.0020996706054532297,0.43631911633893933,0.004829743853718719,0.2866666666666667,5.939289230547254e-27,0.14197120708748615,,,,,,,,,,2.8915775251270757e-11,0.4335645373296677,2.8915775251270757e-11,Baseline
+91,0,Grad. Diff.,Phi,0.0,10,2,1e-05,0.8208616780045351,0.2053333333333333,0.03948523996488023,0.4617578617989009,0.03312670105472504,0.3786869096458096,0.02067738672719293,0.0030140672033289583,0.0005923791732660947,0.4112465508044762,0.010254320002101412,0.5024216524216524,0.04380305738510996,0.4692979336023198,0.029486952804471563,0.38677711002086307,0.01878419442657594,0.005438630481755229,0.0005556600432701432,0.34256549180763196,0.006425791282040907,0.44407238762011636,0.011288158570864326,0.4456124316228852,0.016172136034494928,0.4387009048764408,0.012723258624848553,0.07112488288431025,0.002165385558990518,0.4741629263985171,0.007492712792964969,,,,0.30875170331470164,0.007974400159793121,0.5789688478876374,0.013376387864029145,0.054493439234888544,0.004603772700049135,0.06286718410802472,0.00249583506010431,0.49636067836253955,0.007636791279568331,0.16666666666666666,3.850140759368365e-08,0.31689922480620153,,,,,,,,,,0.0004688948594180668,0.38862474609301456,0.0004688948594180668,Baseline
+92,0,Grad. Diff.,Phi,0.0,10,3,1e-05,0.8208616780045351,0.025,0.012394334537179828,0.4815935750631795,0.03891390096873717,0.38400695068215485,0.028411514664050685,0.0008807482198892126,0.0005780606318251168,0.3382076267332419,0.013329960370499602,0.23817663817663817,0.03609328712275771,0.4788025429991702,0.03245957461517988,0.3843939853103076,0.021060327247136357,0.0037762744455419497,0.0008736819933718429,0.39817194557158064,0.010442966371690138,0.26913251229634727,0.008318145570150776,0.502772110633196,0.019199442290337928,0.11338575553958656,0.007936346771898294,0.048124009138765585,0.002558802040992586,0.3751924199022533,0.008887224559306744,,,,0.1894867784117846,0.004641042318677772,0.47669152933424613,0.016473403722812002,0.0002766024657169499,8.370774640982862e-05,0.0435711624312673,0.0021496425240812313,0.3439928424171195,0.008872932813047263,0.18,0.14165394443314375,0.4510520487264673,,,,,,,,,,0.00011580724071708764,0.13217173107417687,0.00011580724071708764,Baseline
+93,0,Grad. Diff.,Phi,0.0,10,4,1e-05,0.8208616780045351,0.06083333333333333,0.019093330641141205,0.5086347329545251,0.04106627386262414,0.3487585126010581,0.0311229051402676,0.0009206726703364054,0.00028304833011280297,0.20953284692620008,0.016489932417523244,0.34002849002849,0.04041876238615587,0.5447495306271707,0.033931359325418194,0.4064550825116915,0.024244404505141467,0.003870131209698908,0.000495195820981596,0.3127129829424078,0.013231132627163454,0.38118660416278505,0.01141174188129042,0.49680372751005913,0.017118264884523133,0.44515118672144394,0.012320610824723199,0.060515475586569875,0.0019308386607570646,0.4066249338835942,0.011449592642912466,,,,0.09423709669538781,0.004523192731968132,0.5121976536358716,0.01756234639022113,5.079474684466335e-05,3.247775961231264e-05,0.015294580506806769,0.000756985372720552,0.05267286227958383,0.004882019183521703,0.24333333333333335,8.55908179140657e-05,0.36916943521594686,,,,,,,,,,3.3247401264281346e-08,0.25429955690432804,3.3247401264281346e-08,Baseline
+119,0,KL Min.,Phi,0.0,10,0,1e-05,0.7210884353741497,0.41733333333333333,0.04908701817726713,0.4404119900313483,0.03215834584785806,0.3685403748239444,0.018886852683290357,0.006609113947183648,0.00083314965658764,0.32839321242418057,0.002507967748268899,0.7813390313390313,0.036833144291997535,0.49309723511408027,0.029165321907741543,0.40840541183725465,0.0182786916131292,0.008647799939233133,0.0006019463377911001,0.3164046884334776,0.002967030477080494,0.8340946493773876,0.013426858520502996,0.47125500023974304,0.014748205422405927,0.8864616240665817,0.004533328375479302,0.13701555671396073,0.003514771618699596,0.40423482523744636,0.005668013534349268,,,,0.7764363369638583,0.014071456253801585,0.4947696579702449,0.013494912442660918,0.8517714980231927,0.006555103140143843,0.1500083919623026,0.0040543560030796835,0.42862182856275277,0.005687185269494926,0.3466666666666667,8.960455869891988e-43,0.043410852713178294,,,,,,,,,,2.1942743021891237e-16,0.511783767496194,2.1942743021891237e-16,Baseline
+120,0,KL Min.,Phi,0.0,10,1,1e-05,0.7210884353741497,0.06616666666666667,0.02277895414552839,0.38616568889864217,0.032841923580659,0.3373331931585142,0.018648271660188672,0.0009673422788279231,0.0003269511784435408,0.3757633473703473,0.007062844551478875,0.4568376068376069,0.04324292953572399,0.4500329359673502,0.02956541431818185,0.3799713615715442,0.017526960438075313,0.004801713022895753,0.0004947101310173516,0.3286703437685256,0.003873008065887857,0.358224456612624,0.008514703588570245,0.40326034100983643,0.01602569865634584,0.170710124332116,0.00846030855071312,0.054515254890658366,0.0010553167957073715,0.3983812445013232,0.005187913018942307,,,,0.3532902519889898,0.008232457523358213,0.5664525573018395,0.013598115309911316,0.12235810684039583,0.007698740904127741,0.0649760526318002,0.0014454081190137878,0.429791667216399,0.006016338046140314,0.18666666666666668,9.261379512870918e-08,0.3221262458471761,,,,,,,,,,5.522790313356146e-05,0.23172372077326256,5.522790313356146e-05,Baseline
+121,0,KL Min.,Phi,0.0,10,2,1e-05,0.7210884353741497,0.0,0.0,0.3896628430219675,0.040382667395010674,0.29674549248224763,0.029609856180972914,0.0,0.0,0.02605066361846702,0.00033757789187220373,0.0,0.0,0.30882691795012995,0.03233780155076009,0.2354926554575992,0.017661041881035233,0.0,0.0,0.02611884871771877,0.00033386902415561516,0.007550746154505331,0.0010349914605595788,0.15479912781898394,0.013116735104066175,1.0046068308868282e-25,2.4646078942378577e-26,0.001258300749534157,0.00015343091812601938,0.02759032883735345,0.00021697095771110273,,,,0.006485144573910058,0.0009435035460830455,0.714292514930254,0.012802150491243931,4.854365133918027e-26,8.929189246805764e-27,0.0013695072986089475,0.00018652953507862944,0.02772596694393935,0.00021692720249407658,0.24666666666666667,2.1923745611671208e-06,0.34230343300110744,,,,,,,,,,2.0179572425111938e-08,0.0,2.0179572425111938e-08,Baseline
+122,0,KL Min.,Phi,0.0,10,3,1e-05,0.7210884353741497,0.0,0.0,0.44072821233597004,0.04260482389159185,0.2785531708647916,0.032310448587946954,0.0,0.0,0.011330985493835564,1.958102121849729e-05,0.0,0.0,0.35823112850426997,0.03549349991263717,0.24838314796984587,0.018921114350511508,0.0,0.0,0.011379685390984825,3.0772137439828354e-05,0.007550746154505331,0.0010349914605595788,0.1666880102772587,0.014848232404540763,6.649611643545127e-39,2.1976276979304463e-39,0.001087085413713033,0.00013248729005034838,0.011481522077037848,1.091040809265311e-05,,,,0.006485144573910058,0.0009435035460830455,0.6894464972216763,0.014104224091332612,2.3147277267651083e-39,8.501699378930412e-40,0.0012010521474301438,0.0001646876652112906,0.011994085443504188,3.169031440214118e-05,0.3,7.988031105534583e-06,0.3512735326688815,,,,,,,,,,2.5881511422773775e-12,0.0,2.5881511422773775e-12,Baseline
+123,0,KL Min.,Phi,0.0,10,4,1e-05,0.7210884353741497,0.0,0.0,0.46635820448756704,0.04420621513121829,0.24670611228608602,0.033424713350826064,0.0,0.0,0.011330985493835564,1.958102121849729e-05,0.0,0.0,0.42242444283905484,0.03756859490016205,0.2869533480674148,0.024705588796362796,0.0,0.0,0.011379685390984825,3.0772137439828354e-05,0.007550746154505331,0.0010349914605595788,0.17398114915082943,0.015774595323348888,1.3172543247790392e-33,1.128078192351984e-33,0.001087085413713033,0.00013248729005034838,0.011481522077037848,1.091040809265311e-05,,,,0.006485144573910058,0.0009435035460830455,0.6702601806503722,0.014507975683145874,3.466152028939939e-39,2.4482241539901036e-39,0.0012010521474301438,0.0001646876652112906,0.011994085443504188,3.169031440214118e-05,0.31333333333333335,4.564366539733312e-06,0.3473311184939092,,,,,,,,,,2.0608068972918464e-13,0.0,2.0608068972918464e-13,Baseline
+99,0,Pref. Opt.,Phi,0.0,10,0,1e-05,0.8208616780045351,0.38733333333333336,0.04848034069953959,0.43958768915672614,0.03201377473778239,0.36954148073282966,0.018475406909683512,0.0062186222122174225,0.0008318323673224484,0.3296917068571787,0.0046509310873603625,0.7642450142450141,0.03790882473231565,0.5085866828004575,0.027376659012571938,0.4128835041291744,0.017453614159539964,0.008348131177790142,0.0005851803215926627,0.3099638539004192,0.0034217253479417744,0.7897970485636896,0.01560307813848238,0.4610280292306868,0.014411367134427746,0.8766018821342342,0.004091227722166755,0.1273599112562381,0.003687028045594734,0.3973241301372439,0.005886132812497454,,,,0.7561071918107968,0.016287888925169285,0.505780143762839,0.013469338616808978,0.875980368026049,0.004282309480310711,0.14579489306184049,0.004599239936926811,0.4152343327881947,0.005917592510452096,0.32,5.144304720368566e-37,0.07665559246954595,,,,,,,,,,5.563730678470443e-14,0.5046156138615842,5.563730678470443e-14,Baseline
+100,0,Pref. Opt.,Phi,0.0,10,1,1e-05,0.8208616780045351,0.17066666666666666,0.03688572458133378,0.398097593174122,0.030658872972237534,0.3473319068940162,0.01673575152905193,0.0029095484732297246,0.0006580960661434977,0.2779515373235178,0.0034875882262884114,0.6524216524216524,0.04342010748392519,0.5029589294985205,0.0272167573948497,0.4106615313952846,0.01679715874015566,0.007213782213173275,0.0006315423771492203,0.29640589363423786,0.007256977923894145,0.4157369052689159,0.019439495403639942,0.4428018289135021,0.013837380399883631,0.7747471223103995,0.005464012326173342,0.06518609687132093,0.0032763021315533962,0.360046731049611,0.00817335504355518,,,,0.20367182870403827,0.009274833330809624,0.5309647459878414,0.013148473719160551,0.7234846904991572,0.005864907735236472,0.04102579083602729,0.002255860634011684,0.3804965163857859,0.009105379325214965,0.29,3.4286578485665095e-26,0.14739756367663345,,,,,,,,,,1.5989601818532152e-11,0.38858508844208894,1.5989601818532152e-11,Baseline
+101,0,Pref. Opt.,Phi,0.0,10,2,1e-05,0.8208616780045351,0.29233333333333333,0.04491881940456958,0.4087252900067323,0.0310915737552159,0.35329993323454817,0.017383358638716227,0.004919638997249475,0.0008032913787596555,0.2812622421768081,0.003317656232924557,0.7606837606837606,0.03851471583338498,0.49227062858670234,0.028165690638201627,0.41050816200893164,0.017304776647714917,0.008272116019019353,0.0005999502448213067,0.2766736661994572,0.0044089878789167185,0.6400973874874583,0.019440961303790187,0.4601940385344932,0.014036297864505397,0.8343165200314714,0.004909012902157577,0.10097992956048368,0.003742977958814409,0.34440409616126755,0.005067362634500041,,,,0.2752258962662184,0.015184657951135534,0.5125136820862575,0.013201131180235505,0.7803354761928551,0.0055047247911336064,0.05282075998502547,0.003056916532625831,0.2925720656191614,0.003815817406646774,0.31,4.2734211582588426e-33,0.10084163898117386,,,,,,,,,,3.922502017067572e-13,0.4635901617821062,3.922502017067572e-13,Baseline
+102,0,Pref. Opt.,Phi,0.0,10,3,1e-05,0.8208616780045351,0.22566666666666665,0.04087388558735096,0.415320668948717,0.031062911872498312,0.3558490152972197,0.017601857456544336,0.003802229591357634,0.0007264091963216739,0.28037974506404106,0.0036664109838314704,0.7051282051282052,0.04108592314438152,0.4868892295245139,0.028300346655511868,0.4071353723149788,0.01750881824180998,0.007652637976552868,0.0006120018331332855,0.2818804914342013,0.004265775073193291,0.6761552316550627,0.019385788497320533,0.4662565111620517,0.014205346582392659,0.8576960767364884,0.0047351948047892525,0.10642936056056714,0.003772563028343751,0.34187252464356854,0.004481089053299932,,,,0.23005648767467027,0.012309505775143548,0.5063466880646643,0.013362926315265726,0.7774173932003705,0.005687403349703199,0.04364537660671691,0.002299339457067745,0.28993550598053264,0.003031198702827004,0.31,3.076509991615289e-34,0.09364341085271317,,,,,,,,,,3.922502017067572e-13,0.44188907038498515,3.922502017067572e-13,Baseline
+103,0,Pref. Opt.,Phi,0.0,10,4,1e-05,0.8208616780045351,0.22566666666666665,0.04087388558735096,0.41873749533700433,0.031014138839880974,0.3560428742235176,0.01766535626579844,0.00392613552176532,0.0007586720332192074,0.31701491895791006,0.003986008431068629,0.6951566951566951,0.04147789038411901,0.48347158179024324,0.028520063411345657,0.4052351774271735,0.017626395111619356,0.00756923997348644,0.0006164048502494053,0.32018380504074817,0.004923326108424165,0.7317416816694867,0.018545592358354707,0.471358582657205,0.014228447282412016,0.8762493318448833,0.004170960723817527,0.11742815098580654,0.0039300163222826074,0.37899289762204186,0.004846155113022571,,,,0.22542637329599832,0.011624463946222212,0.5048653246444754,0.013454713155325862,0.7796325730927125,0.005701842462918839,0.04350450756625529,0.0023243657135120967,0.3350620955762081,0.003605206488093111,0.3233333333333333,5.527837426332548e-35,0.08901439645625692,,,,,,,,,,2.858944731506222e-14,0.44483865411257695,2.858944731506222e-14,Baseline
+125,0,Retain Model,Phi,0.0,10,-1,2e-05,0.0,0.44733333333333336,0.04950298429938799,0.4559023106732315,0.03283401691458175,0.3764878052915082,0.019616100500433782,0.007157778674970126,0.0008471703859198075,0.35267965882838304,0.004806363653018593,0.7670940170940171,0.03816773858757475,0.5037410710028157,0.029493147306494313,0.4170062423493305,0.018065802584815813,0.008563835771652536,0.0006222200036427265,0.3480993478787522,0.005115672181072033,0.8686838505851442,0.012597149181808459,0.4778830670614996,0.014402799020616376,0.9033232236014582,0.004136077136604402,0.14567073822653545,0.0037584503851931473,0.4521166520885928,0.007857296458691466,,,,0.4330523884261115,0.008902909468669264,0.6364134686518225,0.012814046873094457,0.13676270625058615,0.007359832649756168,0.08092029760683267,0.0017570194044373376,0.46591939546346245,0.00735441333476235,0.0,1.0,0.0,,,,,,,,,,1.0,0.5256206068691371,1.0,Baseline

versions/phi-1p/phi-1p.csv

@@ -0,0 +1,72 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+126,0,Finetune Model,Phi,0.0,0,-1,2e-05,0.0,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.8752167884046598,0.011809089858149302,0.4882534392910444,0.013534006024118014,0.9086918278343834,0.003939262001123021,0.17179131104641562,0.004405566476237556,0.4299893732259693,0.005748680600806638,0.3566666666666667,4.665567748131161e-44,0.0363233665559247,,,,,,,,,,2.4311282147882553e-17,0.515509929080137,2.4311282147882553e-17,Baseline
+25,0,Grad. Ascent,Phi,0.0,1,0,1e-05,0.98989898989899,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.9381956676442949,0.02117638697886323,0.48436721634156044,0.03437105216632326,0.9013096651244134,0.01261339546768661,0.18878779206666457,0.010800242902431123,0.43829137406265495,0.009875004389628838,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.515509929080137,0.00018791118070078278,Baseline
+26,0,Grad. Ascent,Phi,0.0,1,1,1e-05,0.98989898989899,0.3973333333333333,0.04870415050172085,0.44734011995097217,0.03217894764555533,0.37101624483217494,0.018931284114227592,0.006263486182511122,0.0008210066431042056,0.32826270031879445,0.0028133537367042434,0.7574074074074073,0.038158075922171186,0.49499355300913683,0.029162490891262313,0.40906637777543053,0.01830404458250218,0.008317369380519146,0.0005688010295725542,0.3265691141054026,0.005579470119790743,0.8704867907065715,0.012387375926423057,0.47711656069153385,0.01475710586986576,0.9015643248772981,0.003975181537110572,0.14375860868637821,0.0036205137034801857,0.4086015321631815,0.005744145645483118,,,,0.8539212715516609,0.031346063395792045,0.4864554382002444,0.034567232770465084,0.8766605534610836,0.015848473554253645,0.17152510264609178,0.011667532968495263,0.4508845992949766,0.012713303555537679,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.5117557053103856,0.00018791118070078278,Baseline
+27,0,Grad. Ascent,Phi,0.0,1,2,1e-05,0.98989898989899,0.4073333333333333,0.04890628707870714,0.44734570744479357,0.03215277521384867,0.37064624550793046,0.018948252151536123,0.006486464347411385,0.0008332439923098922,0.332025347168284,0.002837707644733655,0.7745014245014245,0.037116781834202695,0.4952346970066556,0.029089850139072643,0.4080242788415114,0.018254755305729533,0.008451255173139164,0.0005623256403373222,0.3237031862965243,0.004885663169096891,0.8635815274703921,0.012617659782456903,0.47731163399674037,0.014730960287802922,0.8993312170707151,0.0040605534893021515,0.14230110953955352,0.0035690176670312482,0.4117371886891391,0.005971239725188091,,,,0.7414372868190965,0.03883903394813422,0.4911711120113319,0.03468085857264063,0.7981197888358412,0.025418043054027256,0.14793387841693728,0.011256772768740979,0.4443630713043298,0.013172887883572593,0.45,1.8189894035458565e-11,0.006097560975609756,,,,,,,,,,0.0005039436209702519,0.5138571534908382,0.0005039436209702519,Baseline
+28,0,Grad. Ascent,Phi,0.0,1,3,1e-05,0.98989898989899,0.4073333333333333,0.04890628707870714,0.4453090971412346,0.03214679909036357,0.36948689439486265,0.018881652942110645,0.006473883016396425,0.0008314731606930938,0.3281906104746231,0.0024681270373660214,0.7813390313390313,0.03683314429199754,0.49232314132576843,0.029233247111684884,0.40750797857211074,0.018307171683717836,0.008677425031675328,0.0006017828672723666,0.32938512423389454,0.00562486241036905,0.8578837952990432,0.012955881614629204,0.47748790205396763,0.014717378374765904,0.8978873109089981,0.004127682283349969,0.14106219269660653,0.0035734771206889383,0.41202942398197234,0.0058623717633421064,,,,0.6960881343400763,0.038258867277542755,0.49247169177473954,0.034655617320946165,0.7495821197535537,0.02962326117259405,0.14016140942194352,0.01143182689343956,0.4423287141379584,0.012955044609784049,0.45,2.546585164964199e-11,0.007317073170731708,,,,,,,,,,0.0005039436209702519,0.5129450960431514,0.0005039436209702519,Baseline
+29,0,Grad. Ascent,Phi,0.0,1,4,1e-05,0.98989898989899,0.38733333333333336,0.04848034069953959,0.4443316895771594,0.032043329642169424,0.3682301294891522,0.01881980183221979,0.00611290271863906,0.0008182277136136137,0.3277660244265019,0.0026729888003580856,0.7727920227920227,0.03738327846205438,0.4901858228452889,0.029408229348403057,0.40689622626949384,0.018389308680389873,0.008612401960069984,0.000606562749119518,0.3217672330646656,0.003740275965288309,0.8435093130431244,0.013116597063190771,0.4775925624555627,0.014708812448580234,0.8916398078711109,0.004364206089106656,0.13810048993949095,0.0034748093630196705,0.41723247189497054,0.006194143622202872,,,,0.5828656648778943,0.03407974327246517,0.49780798020098926,0.0341660430755681,0.6303913996138544,0.03676077325705445,0.11235539435855682,0.0074950402809397,0.4426724916978159,0.0112463326422877,0.425,1.2732925824820995e-10,0.014634146341463415,,,,,,,,,,0.0012708143485281624,0.5073112568192418,0.0012708143485281624,Baseline
+5,0,Grad. Diff.,Phi,0.0,1,0,1e-05,0.9837146980004123,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.9381956676442949,0.02117638697886323,0.48436721634156044,0.03437105216632326,0.9013096651244134,0.01261339546768661,0.18878779206666457,0.010800242902431123,0.43829137406265495,0.009875004389628838,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.515509929080137,0.00018791118070078278,Baseline
+6,0,Grad. Diff.,Phi,0.0,1,1,1e-05,0.9837146980004123,0.4073333333333333,0.04890628707870714,0.44868540865656376,0.0321686527249458,0.37181150454414463,0.018922815592645456,0.006486170192211922,0.000832931697731944,0.32797138285307753,0.0024823590122747016,0.7642450142450141,0.03790882473231565,0.49526140977918076,0.02909737059755208,0.40862760437107676,0.018312350501609138,0.008559684442016689,0.0006101604726989671,0.3266128835403574,0.0056548228302102255,0.8667360576528719,0.012581188337317105,0.4768783677673477,0.014780214016590484,0.9024583220269511,0.003930009140699577,0.1427206220532496,0.003587697149195878,0.4097825055948626,0.005724001958257458,,,,0.8610547059526258,0.0310405134034964,0.4853542302522998,0.03437512758177433,0.8855634203701139,0.01474243252844984,0.17318639481297496,0.011581303572583452,0.44525711226063913,0.011090624516588238,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.5140829815609338,0.00018791118070078278,Baseline
+7,0,Grad. Diff.,Phi,0.0,1,2,1e-05,0.9837146980004123,0.4073333333333333,0.04890628707870714,0.4475376472728773,0.0323074487522032,0.37122099692672955,0.018969719944702083,0.0064734467850549645,0.0008316649722649365,0.32863394937469964,0.002813004092591287,0.7727920227920227,0.03738327846205438,0.49570746550240574,0.029163731393214636,0.40878600243984897,0.01828286383280146,0.008616024632926886,0.0006068853844140269,0.32363669063640094,0.0049508159719739795,0.870504227208116,0.012514553846996589,0.4772347610708645,0.014774596868529624,0.902174565263344,0.003954000940343957,0.14365200200452463,0.0036194175730700105,0.4154716770517733,0.006134132976312476,,,,0.8018708826123622,0.035103288062549996,0.4892833650663837,0.034487712136420406,0.8362948575749034,0.021194321642462405,0.16272345930922244,0.012371673061698818,0.4461470003825648,0.011681306447496029,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.514478369781102,0.00018791118070078278,Baseline
+8,0,Grad. Diff.,Phi,0.0,1,3,1e-05,0.9837146980004123,0.4073333333333333,0.04890628707870714,0.44753904426196073,0.03222423857184303,0.3707737456616037,0.01889144462871969,0.006462987038550368,0.0008305069278006642,0.3306043324483044,0.0028193586252437575,0.7813390313390313,0.03683314429199754,0.49547558185400636,0.029156699866688916,0.4090568740258969,0.018343893935007367,0.00867835383701535,0.000602291642977825,0.3287917419283681,0.005575730510665786,0.8671415676148869,0.012635303023211242,0.4782122389420539,0.014715714848783009,0.9015918463353482,0.004002293558913793,0.1425717949207332,0.0035690264905189287,0.4130029185733472,0.006009914605424333,,,,0.7430213939200827,0.038398736315238255,0.4898765175661513,0.034214931377283896,0.7876055120858971,0.02644006084492684,0.147778321669503,0.011024127953905437,0.44822075074683765,0.013146154080740722,0.45,2.546585164964199e-11,0.007317073170731708,,,,,,,,,,0.0005039436209702519,0.5147932553751199,0.0005039436209702519,Baseline
+9,0,Grad. Diff.,Phi,0.0,1,4,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.44833956134471337,0.03219311487969079,0.3707543500998032,0.018889212393724163,0.006188810217601383,0.0008284624184854022,0.3307671126776658,0.003013009629687307,0.7898860398860398,0.0362573030185129,0.4947600052010745,0.029207971589399993,0.40830379120347654,0.01835265565399645,0.008728220376175523,0.0005977198906415135,0.3224086334097296,0.004156189184388579,0.8679826636769982,0.012350605614753471,0.47914582513239984,0.014712021370872524,0.8999182032989907,0.004089480603664962,0.14269553117576556,0.0035257459887768666,0.4137676859252873,0.0060584645029375704,,,,0.6436270379913573,0.03887201830889552,0.49243789007986544,0.034178894508616185,0.7231324153658734,0.03180027703782736,0.12632750611312157,0.00980220821529207,0.4400912163132471,0.011719872135126382,0.45,3.456079866737127e-11,0.00853658536585366,,,,,,,,,,0.0005039436209702519,0.5114766298813557,0.0005039436209702519,Baseline
+35,0,KL Min.,Phi,0.0,1,0,1e-05,0.9746444032158318,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.9381956676442949,0.02117638697886323,0.48436721634156044,0.03437105216632326,0.9013096651244134,0.01261339546768661,0.18878779206666457,0.010800242902431123,0.43829137406265495,0.009875004389628838,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.515509929080137,0.00018791118070078278,Baseline
+36,0,KL Min.,Phi,0.0,1,1,1e-05,0.9746444032158318,0.3973333333333333,0.04870415050172085,0.44728902446239105,0.03218356795252413,0.37100541871767156,0.01893179859385056,0.006274075944943428,0.000824922030848276,0.3303089219870328,0.002713607859041618,0.7574074074074073,0.038158075922171186,0.49499160868801123,0.029162363299944538,0.4090631750196614,0.018303909839036594,0.008311510352364968,0.0005689692148083968,0.32773223293554377,0.005482191182751713,0.8708212210594464,0.012381207244497535,0.4771104829046241,0.01475989154892623,0.901602017178373,0.003973347728843266,0.14385460262990374,0.003620106691246055,0.4089341145397958,0.005884837429620094,,,,0.8539212715516609,0.031346063395792045,0.48645782090255196,0.03456714217160867,0.8766605534610836,0.015848473554253645,0.17139831956964943,0.011604002688469584,0.44665876925476555,0.011241633000776397,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.5117586083357036,0.00018791118070078278,Baseline
+37,0,KL Min.,Phi,0.0,1,2,1e-05,0.9746444032158318,0.4073333333333333,0.04890628707870714,0.4459501972480129,0.032147250569322675,0.3698524073030104,0.018853838335508723,0.006410138555944562,0.0008236481790441195,0.32919405814551017,0.0023855357854585617,0.7727920227920227,0.03738327846205438,0.4944772525502684,0.029189796852057737,0.40890753209297365,0.018309465969741583,0.008619326709697636,0.000605652987536889,0.3259745482368673,0.005504124688299596,0.8644732670807849,0.012725421137655237,0.4770221582883435,0.01473796126059729,0.8995617144013818,0.004044063305004252,0.1421489980518293,0.0035798163470490397,0.409747891688481,0.005733914865744404,,,,0.7355449835578416,0.03950861903934351,0.4914792311419337,0.034715402996517965,0.7981980067534827,0.025447423032687603,0.14673958568288328,0.011278651585041795,0.4378549070641552,0.011760080697022423,0.45,1.8189894035458565e-11,0.006097560975609756,,,,,,,,,,0.0005039436209702519,0.5134691972534541,0.0005039436209702519,Baseline
+38,0,KL Min.,Phi,0.0,1,3,1e-05,0.9746444032158318,0.3973333333333333,0.04870415050172085,0.4456991947196746,0.03215966277122295,0.36964723281115164,0.018889792071085803,0.006257042590449862,0.0008205103758582331,0.3301645688918783,0.0026483778453465392,0.7813390313390313,0.03683314429199754,0.49303718049924833,0.029316544474557827,0.4080997155333039,0.018335271177574435,0.008678718170859818,0.000602621662205056,0.3258117561281757,0.0056010494054130455,0.8634078684349685,0.012707220158671272,0.4771776461535086,0.014728449040630889,0.8979833941106623,0.004125198628258003,0.14231791646736963,0.003590713570556948,0.4120013750387407,0.0059784558597488725,,,,0.7058307447567751,0.03762467220687855,0.4915664520701643,0.03429830630265117,0.749746098694655,0.029648831961869206,0.14131349156441358,0.011057901423693596,0.4322310136368344,0.010362471288961101,0.45,1.8189894035458565e-11,0.006097560975609756,,,,,,,,,,0.0005039436209702519,0.5116057893453726,0.0005039436209702519,Baseline
+39,0,KL Min.,Phi,0.0,1,4,1e-05,0.9746444032158318,0.37733333333333335,0.04823455597984076,0.4451312466103927,0.032074337973567416,0.36881099308750925,0.018903959324709778,0.0059874249941924895,0.0008190026906871147,0.3303457487722835,0.0028451252827916503,0.7685185185185185,0.03740362832403835,0.4918437433567759,0.029355067089610144,0.40717173960466846,0.018403339661921904,0.008547287120320947,0.000603847838139359,0.32100142283307515,0.003938698868090708,0.8428443007768867,0.013282883318907314,0.4777430419737114,0.014703616309482685,0.8920185483075557,0.004357109300321827,0.13791498221330778,0.0034626333594075798,0.4134893271474381,0.0060851720970190645,,,,0.5782954585385347,0.03430208142559459,0.49893064938975257,0.034342567821562794,0.6305634094200867,0.036773093087640214,0.11157094004096924,0.007548761190745235,0.4460763047379145,0.012491670364386375,0.425,1.000444171950221e-10,0.013414634146341463,,,,,,,,,,0.0012708143485281624,0.5056422614621192,0.0012708143485281624,Baseline
+15,0,Pref. Opt.,Phi,0.0,1,0,1e-05,0.9837146980004123,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.9381956676442949,0.02117638697886323,0.48436721634156044,0.03437105216632326,0.9013096651244134,0.01261339546768661,0.18878779206666457,0.010800242902431123,0.43829137406265495,0.009875004389628838,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.515509929080137,0.00018791118070078278,Baseline
+16,0,Pref. Opt.,Phi,0.0,1,1,1e-05,0.9837146980004123,0.4073333333333333,0.04890628707870714,0.4489850050826217,0.03225845881655124,0.37224482018994254,0.018985125243437023,0.006473743083891557,0.0008315444125398596,0.3254124620029657,0.0026943664665070462,0.7659544159544159,0.037649394138102935,0.4970882841437024,0.029006757595648636,0.4094247685032987,0.01825566212866724,0.008383623032350601,0.0005657777541473935,0.3242876344534,0.0055548040980471424,0.8671142353045616,0.012471023827958234,0.47606098235209154,0.014768102870306712,0.9023179047674869,0.003922789842839779,0.14286495163153254,0.0035401553841250357,0.41037595274329824,0.005839125856205601,,,,0.9330309672201167,0.021853616497567792,0.48643911896540437,0.034277537346453654,0.9010443301295336,0.012574648672749041,0.18783049443545252,0.010811458129562046,0.44451988008365484,0.011218002760991452,0.475,1.2732925824820995e-11,0.004878048780487805,,,,,,,,,,0.00018791118070078278,0.5145665974848272,0.00018791118070078278,Baseline
+17,0,Pref. Opt.,Phi,0.0,1,2,1e-05,0.9837146980004123,0.3973333333333333,0.04870415050172085,0.4491712231703405,0.03226886991221277,0.37279699051781207,0.018991983963438937,0.006344285490875648,0.0008322774860283573,0.3272932411424616,0.0025697205117798574,0.7642450142450141,0.03790882473231565,0.49855809602618845,0.028792933219349295,0.4098851505834903,0.018117937478638112,0.00855662960770205,0.000612820596756875,0.31847697450564455,0.00411682140084272,0.8666505821636726,0.012861245146145787,0.4741905325045548,0.014707545858556728,0.9015666105080429,0.003913986167791284,0.14302421206131977,0.003641820734049634,0.4116082431844605,0.006023237326044905,,,,0.9185294366978219,0.021316348888820707,0.4905423874689709,0.034181738592440364,0.8977405393561181,0.012581508180712856,0.1848156137227071,0.010665564061639772,0.4446187943410747,0.012439499449886359,0.475,1.8189894035458565e-11,0.006097560975609756,,,,,,,,,,0.00018791118070078278,0.5127794902523923,0.00018791118070078278,Baseline
+18,0,Pref. Opt.,Phi,0.0,1,3,1e-05,0.9837146980004123,0.3973333333333333,0.04870415050172085,0.44945734787621616,0.03234404771417288,0.37359707463859704,0.01894855935207623,0.006342658215171831,0.0008318745194273358,0.3246666616822851,0.0024558218476043473,0.7727920227920227,0.03738327846205438,0.4991752890211673,0.028657231340903942,0.41060168729471536,0.017986216686673884,0.00861903389671744,0.0006078251846535709,0.31596479309852454,0.003795369293627395,0.8660650186616514,0.012947218687872044,0.4724868102294998,0.014680718341847771,0.9010237385590695,0.0038858369211263156,0.14267391233686347,0.0036306517956240696,0.40985059525490936,0.006001589859070673,,,,0.9010788605397811,0.02786512361785225,0.4953714647461885,0.03430355713172164,0.8922957468659709,0.012661474480246291,0.18106744471903297,0.011528644483560276,0.447550153698668,0.012946999087382446,0.475,4.547473508864641e-11,0.00975609756097561,,,,,,,,,,0.00018791118070078278,0.5133445153755323,0.00018791118070078278,Baseline
+19,0,Pref. Opt.,Phi,0.0,1,4,1e-05,0.9837146980004123,0.4073333333333333,0.04890628707870714,0.44775042997148523,0.03238472843064173,0.37342411191729574,0.01898011229807771,0.006488728815292577,0.0008338447434264748,0.322111466077318,0.002699263017488665,0.7642450142450141,0.03790882473231565,0.5005958992077298,0.028513844628767247,0.41105922285791363,0.01788232770195742,0.00855034167153587,0.0006109420299516348,0.31216234328703946,0.0029088676961579905,0.8630235409553103,0.013020140276001927,0.4708506097824937,0.014625044357353395,0.8999396728215585,0.003869018215951114,0.14201429377409552,0.003646058278982462,0.4056852456481182,0.005737696809606483,,,,0.8618708658299875,0.03347284506939616,0.4990465502257882,0.034281423153676806,0.8815489828873548,0.012847001873313047,0.17307691454306728,0.011715725848197642,0.4442958540187635,0.014485684318669156,0.475,3.765308065339923e-10,0.020731707317073172,,,,,,,,,,0.00018791118070078278,0.5143193024987673,0.00018791118070078278,Baseline
+41,0,Retain Model,Phi,0.0,1,-1,2e-05,0.0,0.41733333333333333,0.04908701817726713,0.45783926809466224,0.03305397798751837,0.38180200003278386,0.019651959123090125,0.006650119841252343,0.0008361738912389345,0.3360844536600135,0.0030123859366680246,0.7257834757834757,0.039763488344732846,0.49172629935967993,0.029256382358827042,0.40953168852397737,0.018241907846353475,0.008231357710091945,0.000622467436847553,0.32767004075399175,0.0038654559826867027,0.8781841931599393,0.01138950331429817,0.47858936704250027,0.0146725438332836,0.9056413263174269,0.0036558785423746634,0.14638917492890713,0.003590313796402128,0.4530173013101492,0.00782466712167954,,,,0.42706460154385173,0.028397188521646986,0.6613765913805292,0.03507393230874414,0.17076435669886378,0.02994780207067888,0.079180300952986,0.004087766978084058,0.43983005260554914,0.014588202260298236,0.0,1.0,0.0,,,,,,,,,,1.0,0.5176828673725652,1.0,Baseline
+24,0,Grad. Ascent,Phi,0.01,1,0,1e-05,0.98989898989899,0.39233333333333337,0.048334447914700075,0.47106107278003373,0.03215225490833134,0.3819661836885344,0.02464366247814945,0.006317668498978053,0.000835123354007051,0.33554252212504204,0.002709913726566863,0.7642450142450141,0.03790882473231565,0.5001455946443598,0.029188655009533084,0.4228176788373564,0.024780112799567985,0.00860396443668814,0.000624871983420895,0.33092969896178875,0.005380887353128073,0.8696677051904237,0.012341050186972945,0.4736913696419042,0.014947147093686546,0.9023434727049431,0.003921104397976069,0.1430440978881365,0.003509626506330469,0.4228109052199245,0.006154800168940804,,,,0.9273149090847037,0.021480963110959216,0.48644975844069477,0.03467122427212445,0.90172315641436,0.012529015981428812,0.1895832646966106,0.010898608745874642,0.47358596089561955,0.015797952804189035,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5192735229369373,0.00018791118070078278,Baseline
+25,0,Grad. Ascent,Phi,0.01,1,1,1e-05,0.98989898989899,0.3973333333333333,0.04870415050172085,0.4691141904426271,0.032187114431551596,0.38185851674870763,0.024631684319720258,0.00635287414100886,0.0008323238796090722,0.3362393946194996,0.002561648916142886,0.7727920227920227,0.03738327846205438,0.5007502937651985,0.02913699590761685,0.4230345334013543,0.02480236194466246,0.00865986969041309,0.0006201978388291764,0.3297809590558845,0.004773159767438224,0.8647643950421807,0.012574895573603572,0.4738633417290816,0.014919198842056409,0.9016971757706093,0.003960450565372028,0.1420961817001078,0.0035491682338650112,0.42061222293170564,0.006061023340165375,,,,0.8351083204153287,0.03468185604306714,0.48547395327449755,0.0343302859804704,0.8762624998326849,0.015939609912789067,0.16977828225000077,0.011745558994093767,0.4714677044614162,0.014098106109854213,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5202964894841409,0.00018791118070078278,Baseline
+26,0,Grad. Ascent,Phi,0.01,1,2,1e-05,0.98989898989899,0.38733333333333336,0.04848034069953959,0.4692552717598635,0.0319858379756086,0.38084732616315853,0.024653511750053078,0.006217831116610556,0.0008325026644222629,0.3352374736866834,0.0023782089038650246,0.7642450142450141,0.03790882473231565,0.4982178627701232,0.02924159180960339,0.42222187367452574,0.024732882573132414,0.008593509225866842,0.0006238956439960916,0.32738407165812244,0.0039952262494613745,0.8626545173228876,0.012615884749887563,0.47403377079269965,0.014884890626082364,0.8994563658075735,0.0040500242530629865,0.1421624553717516,0.0035612554732744085,0.4248196645920974,0.006288685129704801,,,,0.7491315887684878,0.03840270956453479,0.4914152500022265,0.03473571932502338,0.7987346416621142,0.025483078002970002,0.15227715948351514,0.011429540068171234,0.45893339011908807,0.014332063482746715,0.45,0.00010100967301696073,0.16341463414634147,,,,,,,,,,0.0005039436209702519,0.5171485443679109,0.0005039436209702519,Baseline
+27,0,Grad. Ascent,Phi,0.01,1,3,1e-05,0.98989898989899,0.37733333333333335,0.04823455597984076,0.46827697182259764,0.03199714991555438,0.37986275697090166,0.024683683166899117,0.0060852587694064575,0.0008316088723813273,0.3346540367577501,0.0025102001301302293,0.7813390313390313,0.03683314429199754,0.49758126759199706,0.029290823118665352,0.42291758966523413,0.024768525634996402,0.008711461693312057,0.0006142624589183962,0.3304355112761004,0.00468168862477608,0.8621898803203313,0.012675142116322974,0.4737819144984506,0.014879413937365008,0.8980118565021712,0.004101156933668793,0.141822856706416,0.0035623405861251413,0.425677003820111,0.006287501279962867,,,,0.7035248197390535,0.03785429126606341,0.49219857093975267,0.034533939732104536,0.7496804815311098,0.030025387742131277,0.14326477318298653,0.011183244934636558,0.47462469320937883,0.016061767197491774,0.45,0.0001079662160918815,0.16463414634146342,,,,,,,,,,0.0005039436209702519,0.5155707581476769,0.0005039436209702519,Baseline
+28,0,Grad. Ascent,Phi,0.01,1,4,1e-05,0.98989898989899,0.37733333333333335,0.04823455597984076,0.4683050718887695,0.03196494359019205,0.37973423444654464,0.024712637980204625,0.006080660094815265,0.0008308238664007435,0.33721011519441696,0.0027474074646629534,0.7813390313390313,0.03683314429199754,0.4951969127455589,0.02933253322005141,0.42167850628817516,0.024745665337022785,0.00870503408115977,0.0006147226509369577,0.3262692961252157,0.0036998880937771715,0.8449952295372212,0.012991834440605334,0.47377467878050217,0.014860154734651602,0.8922974386393037,0.00432933862865457,0.13882406585213347,0.0034812570781953154,0.429391288265879,0.006498054955344724,,,,0.5568401869809103,0.03385181282192007,0.49954401650303915,0.03412444059301271,0.6364464792336467,0.03720487155677825,0.10974136698219039,0.00758079850021502,0.4594013242305288,0.01278222966051082,0.425,0.00017029843002092093,0.17317073170731706,,,,,,,,,,0.0012708143485281624,0.5141524904253366,0.0012708143485281624,Baseline
+29,0,Grad. Ascent,Phi,0.01,1,5,1e-05,0.98989898989899,0.38233333333333336,0.04809841902243682,0.4656878562210687,0.03185483678808689,0.37891681270776084,0.024945621711883015,0.0060660514877271705,0.0008179669372596636,0.33686435234975043,0.0029637794986684773,0.7727920227920227,0.03738327846205438,0.49286327339842456,0.029581520933661523,0.4199271006536794,0.024913938281251026,0.00864859818334021,0.0006182947158362087,0.32680656999712626,0.003963069223778722,0.8138522096696973,0.013791463830249668,0.4759427221738433,0.014805837954181122,0.8768447298760198,0.005065601389448835,0.13228278077763803,0.0033774160878568297,0.42518901903854023,0.006238692265172211,,,,0.5153328352580208,0.030093793268626302,0.5125376957409661,0.03376893216030942,0.46593670788860864,0.041332065007430675,0.10100573166688265,0.006268528626701115,0.5008845677285123,0.017679916621148442,0.425,0.0004266080886736745,0.19146341463414634,,,,,,,,,,0.0012708143485281624,0.5120448011393222,0.0012708143485281624,Baseline
+30,0,Grad. Ascent,Phi,0.01,1,6,1e-05,0.98989898989899,0.369,0.04798347779615544,0.4652649824138459,0.031901421227671097,0.37881425137317065,0.025073015405731258,0.005853814781267504,0.0008174372444993994,0.3400373175620246,0.0035082700416259422,0.7813390313390313,0.036833144291997535,0.49237848392853567,0.029663903711107636,0.42021734573649105,0.02501877091901449,0.008703449844656574,0.0006150748448367928,0.3249038894978622,0.0037683053725297254,0.7929022300143219,0.014055765183003346,0.4764762666060773,0.014809247473951015,0.8695189025530045,0.005447226130489343,0.1288666480764426,0.003387143077713062,0.4260365268836295,0.006332620990770888,,,,0.509371521687693,0.029151995412908707,0.5145456065317962,0.03356810827479242,0.41335722637707983,0.041227523729198315,0.10027239473070085,0.006393775965940635,0.5096926945227181,0.019243339211422624,0.425,0.0006032901037542615,0.19878048780487806,,,,,,,,,,0.0012708143485281624,0.5084837586756136,0.0012708143485281624,Baseline
+31,0,Grad. Ascent,Phi,0.01,1,7,1e-05,0.98989898989899,0.369,0.04798347779615544,0.46380898510012486,0.031900447979558225,0.3791614072646656,0.025162238730662718,0.005856244154930032,0.0008203253717020242,0.33916173415891043,0.004058804347582513,0.7642450142450141,0.037908824732315646,0.4901217094973814,0.029818007400573093,0.41949967720473597,0.025133885110752527,0.008511006670189884,0.0006210087261035859,0.32756302010210164,0.003869938105448067,0.7775199360472875,0.01416553158908383,0.47750847119003215,0.014804619330946758,0.8620642150080514,0.00584595470538699,0.12581050159259718,0.003322329956915689,0.4289844186852424,0.006312337709294942,,,,0.5015612493487781,0.0278234082958558,0.5168351848342064,0.03364483803027,0.3712698597659793,0.04043530489302637,0.09924327494690141,0.006106059210917637,0.5257463841349544,0.020700739696875955,0.425,0.0006754665791959269,0.20121951219512196,,,,,,,,,,0.0012708143485281624,0.5062886806507887,0.0012708143485281624,Baseline
+32,0,Grad. Ascent,Phi,0.01,1,8,1e-05,0.98989898989899,0.4023333333333333,0.04854852825740032,0.46552289532755425,0.03196817915544298,0.3801214868668356,0.02523734931086047,0.006384576048500103,0.0008274180603713374,0.3408106871651121,0.004508380208437795,0.7642450142450141,0.03790882473231565,0.490037686865792,0.029673226078079068,0.4191095735138308,0.025259609291243903,0.00846994361911571,0.000617609813987176,0.32563031783573965,0.00379815633643885,0.7563971390426154,0.01421262148052474,0.47775131017565314,0.014832564076398162,0.8524204105886942,0.006292773672531188,0.12214556905522372,0.003212424337827836,0.43608762263277623,0.006794768504197196,,,,0.49666270694254255,0.027391970438179983,0.5208033131595877,0.03376144930648525,0.32692163167853544,0.03914980832441415,0.09730839092564955,0.005738286470631208,0.5080816158264476,0.018800927977896627,0.4,0.0009411722930963151,0.20853658536585365,,,,,,,,,,0.003018184077228396,0.5117172322110884,0.003018184077228396,Baseline
+33,0,Grad. Ascent,Phi,0.01,1,9,1e-05,0.98989898989899,0.41233333333333333,0.048740949259120245,0.4660208651618553,0.031988097787335926,0.3798812250358469,0.025340040388076003,0.006533325218413113,0.000827183242722597,0.3453007870486442,0.0036814273920029736,0.753133903133903,0.03816316425302378,0.49079396078693677,0.029671099859135464,0.4193018847778626,0.025285792610138837,0.008167497737694757,0.0005681246260849913,0.3265673756532271,0.003721167893917763,0.7517134745572206,0.014099803370011875,0.4783700858431125,0.014825966205242552,0.8457655076459193,0.006562637938185463,0.12106966510374655,0.003178208878725879,0.4317915984944247,0.0062878367744713315,,,,0.4834522904351621,0.02875883671386502,0.5234591411139148,0.03421207858807924,0.30178133518688965,0.0381847480450477,0.09502896438292638,0.006041381653234637,0.5126974138957279,0.020110106280951858,0.375,0.0009411722930963151,0.20853658536585365,,,,,,,,,,0.006760732303569208,0.5126232355411223,0.006760732303569208,Baseline
+34,0,Grad. Ascent,Phi,0.01,1,10,1e-05,0.98989898989899,0.4023333333333333,0.04854852825740032,0.4650549010452283,0.03212385949330844,0.3799928398269198,0.02538293333575537,0.006400086307343885,0.000827009349836385,0.3488285797583384,0.004296969570893441,0.7445868945868945,0.03864064861949842,0.488464404843072,0.029667727906935563,0.4185716439853416,0.02531846771046855,0.008031918779310879,0.0005685225245077936,0.32584154727641723,0.0039653516234121645,0.7399096265931445,0.014395047272774304,0.47821844276364117,0.01483165590457743,0.8409361401033726,0.006747677816660936,0.11890414606340625,0.003176970856523821,0.4324305244270342,0.00637197217123034,,,,0.4883771859419541,0.02903919470592847,0.5255092976650884,0.034336805461449973,0.2847066619636903,0.03725378117308237,0.09543565563723652,0.005945785400189619,0.4996831000827912,0.017437650222351688,0.375,0.0011671793090499705,0.21341463414634146,,,,,,,,,,0.006760732303569208,0.5090926373156979,0.006760732303569208,Baseline
+35,0,Grad. Ascent,Phi,0.01,1,11,1e-05,0.98989898989899,0.39233333333333337,0.048334447914700075,0.46401088646252103,0.03204591634547726,0.37938169655579146,0.025424564053503462,0.006256167693829846,0.0008262842753896119,0.34783696704184797,0.004607448396649673,0.7588319088319088,0.03804802946912591,0.48915999415287337,0.029684563730988857,0.41914833788976075,0.02537172024478071,0.008269045528105144,0.0005753880266182705,0.3270834283739693,0.003916501463494453,0.7338676926209406,0.014469389711262473,0.478601570751399,0.01478846406436839,0.8381726077881146,0.006824228899375792,0.11752007670397385,0.003125095662126394,0.4298467666485135,0.006180955822154491,,,,0.485157350072015,0.02898186266173905,0.5250209084904061,0.0342005043558565,0.2754518278622501,0.03676025171889327,0.0952382115721843,0.00615512656983899,0.5026880617758069,0.01833831324703816,0.375,0.0012308068326092325,0.2146341463414634,,,,,,,,,,0.006760732303569208,0.5075310717913282,0.006760732303569208,Baseline
+0,0,Grad. Diff.,Phi,0.01,1,0,1e-05,0.9837146980004123,0.39233333333333337,0.048334447914700075,0.47106107278003373,0.03215225490833134,0.3819661836885344,0.02464366247814945,0.006317668498978053,0.000835123354007051,0.33554252212504204,0.002709913726566863,0.7642450142450141,0.03790882473231565,0.5001455946443598,0.029188655009533084,0.4228176788373564,0.024780112799567985,0.00860396443668814,0.000624871983420895,0.33092969896178875,0.005380887353128073,0.8696677051904237,0.012341050186972945,0.4736913696419042,0.014947147093686546,0.9023434727049431,0.003921104397976069,0.1430440978881365,0.003509626506330469,0.4228109052199245,0.006154800168940804,,,,0.9273149090847037,0.021480963110959216,0.48644975844069477,0.03467122427212445,0.90172315641436,0.012529015981428812,0.1895832646966106,0.010898608745874642,0.47358596089561955,0.015797952804189035,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5192735229369373,0.00018791118070078278,Baseline
+1,0,Grad. Diff.,Phi,0.01,1,1,1e-05,0.9837146980004123,0.38233333333333336,0.04809841902243682,0.46968334594275984,0.032090997545820794,0.381847876509468,0.02467010205693857,0.006159696245118596,0.0008311943035200422,0.3371039891249383,0.0026525736415314548,0.7727920227920227,0.03738327846205438,0.5016926787037962,0.029133357732067787,0.4230283644434858,0.024731981862292477,0.008663408793937894,0.0006214673844918737,0.32824241196574044,0.003994969864642441,0.8703212265914079,0.012501140518470311,0.4739276374058327,0.01490699996679798,0.902434282168424,0.003942863965509323,0.14344023683891427,0.003591285328726514,0.4189041499655849,0.005876073842655799,,,,0.8705028003226761,0.030852819399580747,0.48625888864412287,0.03447718096555444,0.8849568967379842,0.014797619920192982,0.17777903343646856,0.011637605234302415,0.46545038878090406,0.013422994194174726,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5177839372824619,0.00018791118070078278,Baseline
+2,0,Grad. Diff.,Phi,0.01,1,2,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.4703728775015927,0.0320527949388215,0.3824314706212928,0.024673810217136435,0.006221825349757508,0.000832147843740073,0.336329595832092,0.0024501646757135384,0.7727920227920227,0.03738327846205438,0.5004708116545773,0.02909784139710554,0.423022449824084,0.024662796663197217,0.008670261906583973,0.000623584465254466,0.3235866877831451,0.003953841042990771,0.8744727303621407,0.012241106790960918,0.4746964922821466,0.014910390206772848,0.9025004656237983,0.003927597783213117,0.14397385401977397,0.003562840331328352,0.42303128871336776,0.006195569222778232,,,,0.7852113444990887,0.036251798787981196,0.4887747639147433,0.03461355221828389,0.8362468851456917,0.02132123357410361,0.1608663858198558,0.012115116165264516,0.46046654126642467,0.01240314003390908,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5191259657408259,0.00018791118070078278,Baseline
+3,0,Grad. Diff.,Phi,0.01,1,3,1e-05,0.9837146980004123,0.39233333333333337,0.048334447914700075,0.4699585669098189,0.03203610005510997,0.38144833107899073,0.02472764517316665,0.0062992845215982165,0.0008312532164697153,0.3374144421431132,0.002511531816578193,0.7784900284900286,0.037239813459932845,0.4991819777882713,0.02917688956035419,0.4224464880250402,0.024699454034639873,0.008665010087211966,0.0006209591957232924,0.32419314896048873,0.003859775907545111,0.8740983501403523,0.012255270271987388,0.4750030411171504,0.014911024137964728,0.9022497656883485,0.0039685062726749374,0.14432359885281365,0.003601994076317492,0.4232251522824819,0.00610813739317317,,,,0.7387187502644437,0.03919762687707207,0.490245758064806,0.03448691361268974,0.7880355591132844,0.026749027643331747,0.15065448525163863,0.011756876210525057,0.47555152932540884,0.01604736794469087,0.45,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.0005039436209702519,0.5199038668033013,0.0005039436209702519,Baseline
+4,0,Grad. Diff.,Phi,0.01,1,4,1e-05,0.9837146980004123,0.38233333333333336,0.04809841902243682,0.4705764594491274,0.0319648659709911,0.3825528276713364,0.02480701079605742,0.006143941183133531,0.0008279428396022854,0.3378518258530328,0.0026464069491313946,0.7699430199430201,0.037778470870230174,0.5014614894007436,0.029005221966733177,0.423186178456126,0.024681407372603416,0.0086028667289517,0.0006242725925294986,0.326053296455976,0.003726033704069992,0.8757945872055253,0.01217638844416652,0.47549030362875083,0.014895323836640011,0.9015248126690938,0.004027484971777941,0.14451407390011833,0.003597347600140643,0.426601342395262,0.006306539305783168,,,,0.6402682481532161,0.037919366242965644,0.49278564398751845,0.034055791375969355,0.7225704019845642,0.03242466112531492,0.12708923876890624,0.009576405917012544,0.4731563811860505,0.016911158951061364,0.45,0.00012320928908593487,0.16707317073170733,,,,,,,,,,0.0005039436209702519,0.5182919935140732,0.0005039436209702519,Baseline
+5,0,Grad. Diff.,Phi,0.01,1,5,1e-05,0.9837146980004123,0.3706666666666667,0.04782421985238585,0.47128105881009524,0.03193034016761994,0.3830845279720404,0.02499857604717198,0.005939163832323779,0.0008180102100740433,0.33759430692944276,0.0030307298385894863,0.7784900284900286,0.037239813459932845,0.5003639224751233,0.02915568967360509,0.42286069752968514,0.024782447200514326,0.00865851078822668,0.0006206082044194495,0.3255564554178044,0.003910207417164235,0.8478222231029351,0.013603098993708956,0.4781012833050873,0.014815083376959382,0.8966624785872678,0.004279793902957413,0.13907025266820702,0.0036218286877939517,0.4218975966327066,0.005930153777455233,,,,0.557328597433796,0.033705566322635855,0.5034566232167167,0.03368701147588357,0.5690474407614421,0.04051216540659692,0.10917030370732084,0.00725251698026869,0.4621163743379803,0.014753892439507433,0.425,0.00023327588314714376,0.17926829268292682,,,,,,,,,,0.0012708143485281624,0.5153345679144592,0.0012708143485281624,Baseline
+6,0,Grad. Diff.,Phi,0.01,1,6,1e-05,0.9837146980004123,0.3856666666666667,0.04794603916024768,0.47007009529477706,0.031950942192174994,0.3839120761469267,0.025128620523099183,0.006220921033695498,0.0008265366987434795,0.34108845725514575,0.003007195755640496,0.7699430199430201,0.037778470870230174,0.4980599052346513,0.02940122031640498,0.4226061959725478,0.02483302858172543,0.00854245962617961,0.0006231456373436992,0.32658552433065685,0.0038634158452649043,0.8438380987393276,0.013496047641892009,0.4779487231142976,0.014862606608793589,0.8924592383031154,0.004521642263258704,0.1381472323152376,0.003578483430072228,0.42468041161765124,0.006259129823076829,,,,0.5370583389304254,0.033686601442450316,0.5061855038042944,0.033122411432869814,0.5035395261921789,0.041558663624014704,0.10590503262348419,0.007226458246234129,0.47952431303665166,0.01489658634888218,0.425,0.0003167852501064772,0.18536585365853658,,,,,,,,,,0.0012708143485281624,0.5173689166403966,0.0012708143485281624,Baseline
+7,0,Grad. Diff.,Phi,0.01,1,7,1e-05,0.9837146980004123,0.3756666666666667,0.04769397160069877,0.46926416702229956,0.03210326930182116,0.3834747710810494,0.02512357170259834,0.006001691670560483,0.0008138220799669095,0.33752493349478774,0.003106201419656793,0.7727920227920227,0.03738327846205438,0.5008415582080639,0.02920305318115105,0.42321147083835314,0.024848739404839743,0.008580025154071576,0.0006175681433410414,0.32853235683946813,0.0038851656182212987,0.8348958609916904,0.013784928589479424,0.47971013498012477,0.014797511135517685,0.8877683574873093,0.0047795885003666366,0.13641923747724238,0.003550323280088861,0.4230205045670796,0.006113236381716485,,,,0.5113520274160418,0.029429310821373244,0.5112281781666459,0.0334362313783791,0.4552004489928631,0.04164071708861998,0.10030164821097332,0.006230769922786472,0.47774846548231087,0.015754486172814514,0.425,0.0004523318730207393,0.1926829268292683,,,,,,,,,,0.0012708143485281624,0.5153764404003137,0.0012708143485281624,Baseline
+8,0,Grad. Diff.,Phi,0.01,1,8,1e-05,0.9837146980004123,0.3756666666666667,0.04769397160069877,0.4708028818403621,0.03204800835269835,0.3838930224894358,0.02521831559041362,0.0060084921389601705,0.0008142611806819213,0.3397985630665407,0.002976638883970388,0.7742165242165243,0.03726587507540985,0.49890412368352555,0.02919099559118092,0.42337931256790934,0.024922389964410506,0.008602634219539315,0.000620598510479757,0.3285331653120452,0.0039213635475069215,0.819381647931274,0.01420904976176903,0.4795627140875673,0.014821141264367179,0.8820885288132968,0.00508686020907914,0.13392740385447738,0.0035777805198427526,0.42939863845526977,0.006246870571576884,,,,0.5089654323788317,0.029584197091004117,0.5142076536424027,0.0333800229984447,0.40954934299039447,0.04124621741408505,0.10002834278526165,0.006638175629868696,0.4804069331934361,0.015280365748356681,0.425,0.0006384575062838849,0.2,,,,,,,,,,0.0012708143485281624,0.5146337212713537,0.0012708143485281624,Baseline
+9,0,Grad. Diff.,Phi,0.01,1,9,1e-05,0.9837146980004123,0.3756666666666667,0.04769397160069877,0.4698512012533962,0.03202472486620743,0.38364630569101493,0.025300293697908482,0.006001080793765982,0.0008141324123642806,0.34137724659636265,0.003384219753711942,0.7588319088319088,0.03804802946912591,0.4975801689776138,0.029320968299058538,0.4221853551608589,0.024995577535612205,0.008215678817529366,0.0005731752500952975,0.32823245330474343,0.003969969474006614,0.8145649037098377,0.014156117316800805,0.47951620524248084,0.014813540099953067,0.8784247142016177,0.005202634556646482,0.1328448806829067,0.0034920518160556618,0.4227935189689803,0.006018675341135678,,,,0.5014111328873474,0.029142692502568398,0.5155578454835973,0.03348758545196584,0.37913454004958635,0.04047901318510703,0.09790399885340889,0.006144879312065473,0.48492949441003025,0.01598944683113668,0.425,0.0008912169196264585,0.2073170731707317,,,,,,,,,,0.0012708143485281624,0.5129815561362334,0.0012708143485281624,Baseline
+10,0,Grad. Diff.,Phi,0.01,1,10,1e-05,0.9837146980004123,0.36233333333333334,0.04755917809293407,0.4713239611853811,0.032066896152610325,0.3840323283024767,0.025344277054830743,0.0057955804578634475,0.0008137616259436301,0.34146940812855847,0.0032611904777300493,0.7417378917378917,0.03900535375717249,0.49793966453022287,0.029483818133792535,0.42214168853126804,0.025028793164267465,0.008089575704980942,0.0005811417280139287,0.3278978723777728,0.0038419478489272107,0.8124407421156895,0.014168072739714283,0.4788904510015802,0.014864696743006254,0.8757016904555477,0.005283866173338976,0.13226258008831934,0.0034794998499216904,0.42750516995121834,0.0061265625749422035,,,,0.5015227312554328,0.02968421513533533,0.5168789147318157,0.033244221766724744,0.3586088424346507,0.03967827594186178,0.09816558721513272,0.0064528517732610705,0.4882662319364609,0.01674548580066215,0.425,0.0008912169196264585,0.2073170731707317,,,,,,,,,,0.0012708143485281624,0.5092855757778905,0.0012708143485281624,Baseline
+11,0,Grad. Diff.,Phi,0.01,1,11,1e-05,0.9837146980004123,0.35233333333333333,0.047255199418200236,0.4715890418438285,0.03201327690465643,0.3838378517657449,0.02538576252212471,0.00565021195780269,0.0008109601259962053,0.3429432698150645,0.0030628422879978756,0.7485754985754987,0.03878537265351063,0.49646859538782306,0.029407484196674306,0.4217837770945688,0.02501448340157833,0.008297755529560106,0.0006273437564698512,0.3300946694011867,0.004112029712803115,0.8100748286809943,0.014397117725099128,0.47894555763379193,0.014881835423776408,0.8741562583091229,0.005327642090610837,0.1316713262378962,0.003475151379026987,0.4308599680413131,0.006297105384855297,,,,0.4974402545923109,0.028465845359835783,0.5162766688657138,0.033124250041306,0.34863903437286214,0.03939242427633976,0.09766389168200143,0.006202908456664105,0.5003716191752916,0.018095141371351042,0.4,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.003018184077228396,0.507005341274042,0.003018184077228396,Baseline
+36,0,KL Min.,Phi,0.01,1,0,1e-05,0.9746444032158318,0.39233333333333337,0.048334447914700075,0.47106107278003373,0.03215225490833134,0.3819661836885344,0.02464366247814945,0.006317668498978053,0.000835123354007051,0.33554252212504204,0.002709913726566863,0.7642450142450141,0.03790882473231565,0.5001455946443598,0.029188655009533084,0.4228176788373564,0.024780112799567985,0.00860396443668814,0.000624871983420895,0.33092969896178875,0.005380887353128073,0.8696677051904237,0.012341050186972945,0.4736913696419042,0.014947147093686546,0.9023434727049431,0.003921104397976069,0.1430440978881365,0.003509626506330469,0.4228109052199245,0.006154800168940804,,,,0.9273149090847037,0.021480963110959216,0.48644975844069477,0.03467122427212445,0.90172315641436,0.012529015981428812,0.1895832646966106,0.010898608745874642,0.47358596089561955,0.015797952804189035,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5192735229369373,0.00018791118070078278,Baseline
+37,0,KL Min.,Phi,0.01,1,1,1e-05,0.9746444032158318,0.3973333333333333,0.04870415050172085,0.4693683193146157,0.03216935613071336,0.3817674298716135,0.024626470561223326,0.006390538611325378,0.0008388628514610743,0.33480051600285676,0.0025438726089879573,0.7727920227920227,0.03738327846205438,0.5007687657393778,0.029138492383236157,0.42304214922377037,0.02480040947261073,0.008660340883445968,0.000619981268040081,0.3312178344724497,0.0049101428306495285,0.8645210339136452,0.012589328821100242,0.473851028865007,0.014919925134490447,0.9016231488419769,0.0039682426406496395,0.14212708406234265,0.0035541811838022998,0.421349039779618,0.006102588568848379,,,,0.8351083204153287,0.03468185604306714,0.48543553801035,0.034325594606598536,0.8763363934438217,0.0159458536978417,0.16987585909909497,0.011761166828691388,0.472490726462257,0.014034876494622938,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5203017273761114,0.00018791118070078278,Baseline
+38,0,KL Min.,Phi,0.01,1,2,1e-05,0.9746444032158318,0.38733333333333336,0.04848034069953959,0.46748527400767054,0.03203512671438045,0.38065863279959977,0.024624176405202154,0.0062145248809819245,0.0008314229951916702,0.33668006693349534,0.002387081424047051,0.7898860398860398,0.0362573030185129,0.49842215078902413,0.029176774161061732,0.4228080976369466,0.024737271998204663,0.008769453935932467,0.0006105723088491857,0.3329737898703605,0.005301080113175988,0.8667498013677366,0.01249667760447722,0.47325626179752095,0.014898746117798636,0.8994660136680576,0.004038774038964199,0.14259396730182083,0.003560234568630265,0.42631908739191315,0.006362839514928002,,,,0.752069610176904,0.03837578051401302,0.4919313341213655,0.03508175168424802,0.7987979280454878,0.025502376972673216,0.15321496287413355,0.011640477582715383,0.46766347371848555,0.015168292902195955,0.45,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.0005039436209702519,0.5183170841088887,0.0005039436209702519,Baseline
+39,0,KL Min.,Phi,0.01,1,3,1e-05,0.9746444032158318,0.37733333333333335,0.04823455597984076,0.4670192564577889,0.03187794707624757,0.38043078819818943,0.024669505927405747,0.006075667945896106,0.0008307087490943248,0.3370361627465048,0.002645466989972904,0.7898860398860398,0.0362573030185129,0.49752020923291157,0.02936384051030043,0.42208745116685237,0.02477573645796673,0.008773234610149795,0.0006101807257702697,0.33175519217081495,0.004729086340890961,0.8636567889574792,0.012676600757409458,0.4734494368011259,0.014915329036777482,0.8981140305817465,0.004108412269362885,0.14235407396042657,0.0035863986465511106,0.4284877345945476,0.006459749638345023,,,,0.7000953819483986,0.03799326966028634,0.4938846300093765,0.034667360672219046,0.7497184656319902,0.029945045610757073,0.14239421378354944,0.011183943428077454,0.4829540089176213,0.017422659747347996,0.45,0.00011535796875250526,0.16585365853658537,,,,,,,,,,0.0005039436209702519,0.5157996301408445,0.0005039436209702519,Baseline
+40,0,KL Min.,Phi,0.01,1,4,1e-05,0.9746444032158318,0.37733333333333335,0.04823455597984076,0.46672308708499133,0.031981336942478435,0.3799535521757767,0.024814816931711614,0.006079820500298093,0.0008302257569260186,0.3379250594616313,0.002751551207000693,0.7813390313390313,0.03683314429199754,0.4961550482011312,0.02929308028349156,0.42104554299968877,0.024772969981370938,0.008722978901266755,0.0006182275761392474,0.3275599037441755,0.0038197059141391386,0.843758528506807,0.012992186300695253,0.4747654412752712,0.01484925957484625,0.8926066254206179,0.004321110182493689,0.13840243569608365,0.003508624046049031,0.4260111702043169,0.006258499735065518,,,,0.5594639659639565,0.03330123884152528,0.4996118084164255,0.034234509461641575,0.6366187874928668,0.03711331276091983,0.11080302262215928,0.007923041826677397,0.4724597423141951,0.014894910072019023,0.425,0.0001597403588675661,0.1719512195121951,,,,,,,,,,0.0012708143485281624,0.5140842120174873,0.0012708143485281624,Baseline
+41,0,KL Min.,Phi,0.01,1,5,1e-05,0.9746444032158318,0.38233333333333336,0.04809841902243682,0.46661963081131397,0.03175671051154822,0.3795513471467959,0.024979401303422254,0.006066683635827918,0.0008171283139956043,0.34078198352722955,0.003099265230063683,0.7984330484330483,0.03565450910887651,0.49506600533079254,0.02931052514640782,0.421653539241696,0.02495105297223016,0.008820582357258034,0.0006052592205008228,0.3239887332242519,0.0028279105309601455,0.8124993006855429,0.013949978972286607,0.4760999770578121,0.014791909571315773,0.8775803095771189,0.005051948103917992,0.13229293131648526,0.0034322194625840525,0.42539499394789415,0.006257874766935679,,,,0.5144388174484309,0.0313943612693118,0.5115159087167633,0.033457324804672194,0.4664356404676848,0.04140708857548905,0.10121536531974532,0.00685102362656209,0.4950466571751274,0.018307644165936474,0.425,0.0004022172452096129,0.1902439024390244,,,,,,,,,,0.0012708143485281624,0.5140522164958851,0.0012708143485281624,Baseline
+42,0,KL Min.,Phi,0.01,1,6,1e-05,0.9746444032158318,0.369,0.04798347779615544,0.4656494734289401,0.03194461506974083,0.380035834037453,0.02506666391732315,0.005840816376111096,0.0008160691560246799,0.33752179555802736,0.002934774011611804,0.7927350427350427,0.03583167667835136,0.4922652421235348,0.029544619538345872,0.4205361598415399,0.025052454791332505,0.008714977398435388,0.0006015236136412495,0.3237006043380445,0.0029225498052488706,0.7990553414012964,0.014128445944383354,0.47651541523777496,0.01482555504957792,0.8703947691873343,0.00544939660195073,0.1298052519382808,0.003373667807252486,0.43012985259044834,0.006581205541459725,,,,0.511217871542697,0.03024107132098399,0.5155455707901652,0.033715958654653044,0.41400579997261067,0.04127612088322259,0.10032415846044249,0.006322614565791503,0.5012144492055711,0.018695379392584057,0.425,0.0005381573264457984,0.19634146341463415,,,,,,,,,,0.0012708143485281624,0.5096654291871373,0.0012708143485281624,Baseline
+43,0,KL Min.,Phi,0.01,1,7,1e-05,0.9746444032158318,0.37233333333333335,0.047840116718887375,0.4622086892084715,0.03199169604204048,0.3796236596613422,0.025121422357595923,0.005918266421514356,0.0008142771112026905,0.3374200466826558,0.002896220507286891,0.7670940170940171,0.03751873572115046,0.4929355904548995,0.02948922504160227,0.4201856108010598,0.02510117351778158,0.008514705988737476,0.0006107967739492087,0.32601411182971723,0.0037853661197566808,0.7779557174266457,0.014158678704467744,0.47697211458362704,0.014854839250595327,0.8632711128767464,0.005830753155571272,0.1260960085844987,0.0033325450481360036,0.4300651397049737,0.006542126474561786,,,,0.5005932994814926,0.027310286083198406,0.5187947728757729,0.03399374741069954,0.3716535667341191,0.040549083361672324,0.0991005480036768,0.005886185801167351,0.5072595254021156,0.019851850953784837,0.425,0.000714401952791377,0.20243902439024392,,,,,,,,,,0.0012708143485281624,0.507441746730203,0.0012708143485281624,Baseline
+44,0,KL Min.,Phi,0.01,1,8,1e-05,0.9746444032158318,0.39233333333333337,0.048334447914700075,0.46433918523143,0.03185891200226397,0.37975777111627634,0.02529275745010859,0.006274214106174164,0.0008304786615280901,0.34055965315926606,0.0028979032602286132,0.7685185185185185,0.03740362832403835,0.49013216054756503,0.029616552478116023,0.420079201803198,0.025229286487929155,0.008463921352537722,0.0006106387345924967,0.3263135266733187,0.003836909285912203,0.7568481585708866,0.01428971138961719,0.47738364932451705,0.01483178056276175,0.8535920668796955,0.006278443917368893,0.12186146306551661,0.0032260103838620395,0.42534052326820493,0.006182603722016793,,,,0.4905318770869142,0.027825397266743865,0.5203624555920119,0.03382456315074457,0.32750453983977823,0.03920438359166655,0.09598607517931182,0.005789514203038146,0.49329718378298704,0.017398100893484633,0.4,0.0009411722930963151,0.20853658536585365,,,,,,,,,,0.003018184077228396,0.5100530604240106,0.003018184077228396,Baseline
+45,0,KL Min.,Phi,0.01,1,9,1e-05,0.9746444032158318,0.39233333333333337,0.048334447914700075,0.46569137919568343,0.03192442696925184,0.37977323601430657,0.02533463536096106,0.006261764168202744,0.0008275872174925255,0.34330013308815077,0.0038430642521031567,0.7599715099715099,0.03792058995790417,0.489500493082149,0.02963028382198511,0.41998566145768756,0.025289424379595814,0.008347802745736842,0.000611727684854704,0.3281775783401527,0.0038902765243317605,0.7501548650710678,0.014276290387539699,0.4777142889338256,0.014825878420149633,0.8474191916120225,0.006525903164386064,0.1211424392898338,0.0032203248708548805,0.4350469587170104,0.006737254567204074,,,,0.488441220207951,0.028574686249393265,0.521977224082869,0.03369426679582365,0.3018317889571157,0.03817504902796229,0.09574740993990964,0.0059271846246293945,0.5003407584459282,0.018066735824931,0.4,0.0009936344267771346,0.2097560975609756,,,,,,,,,,0.003018184077228396,0.5091784358553739,0.003018184077228396,Baseline
+46,0,KL Min.,Phi,0.01,1,10,1e-05,0.9746444032158318,0.37233333333333335,0.047840116718887375,0.46591443553161893,0.0319913723462668,0.379618742111869,0.02537725587638941,0.005977889141912617,0.0008234929393976787,0.34288770372834887,0.003671966166317685,0.7431623931623931,0.038266114519774866,0.4888144622959025,0.02967804433765736,0.4195335201672202,0.02532399283402574,0.008033992358925587,0.0005620294604403494,0.3262256238100076,0.0031889824880277514,0.7433964244006845,0.014284550257803154,0.4781687679131385,0.014818208711896609,0.843157627632441,0.006691027247235992,0.11959925327649787,0.0031585335668434804,0.4309199036434964,0.006452711151863429,,,,0.4860688843218532,0.029150676278673963,0.5243762523852766,0.03412291962867894,0.2855184085621543,0.03736175091035647,0.09534122999291009,0.006092636670118598,0.5036426888541229,0.018991327618991145,0.375,0.0010487130839464953,0.21097560975609755,,,,,,,,,,0.006760732303569208,0.5038128040005867,0.006760732303569208,Baseline
+47,0,KL Min.,Phi,0.01,1,11,1e-05,0.9746444032158318,0.3973333333333333,0.048182872741143454,0.46580599720278554,0.032186486911030626,0.38140561309676224,0.02539721839626697,0.0063224024960747485,0.0008237694967847481,0.34509067895439904,0.003473554858993239,0.7317663817663818,0.03908102870464613,0.49018121545712556,0.029547077607603375,0.4194701731983185,0.02529795907227767,0.007917888704511301,0.0005719818945835358,0.3280420737581117,0.0038464452946215246,0.7362316848590984,0.014445586926128353,0.4782364294459663,0.014834686414403981,0.8406052036779104,0.006769158027789019,0.11790352446092378,0.0031045118269796583,0.43333676680868793,0.006437032587517537,,,,0.48649233762976146,0.02902002241318492,0.5249621584390722,0.0337709030119393,0.2767055241480099,0.03687226372620335,0.09592190410580832,0.006118309796687845,0.5185498881897033,0.020222913736717862,0.375,0.0012308068326092325,0.2146341463414634,,,,,,,,,,0.006760732303569208,0.5080452710322519,0.006760732303569208,Baseline
+12,0,Pref. Opt.,Phi,0.01,1,0,1e-05,0.9837146980004123,0.39233333333333337,0.048334447914700075,0.47106107278003373,0.03215225490833134,0.3819661836885344,0.02464366247814945,0.006317668498978053,0.000835123354007051,0.33554252212504204,0.002709913726566863,0.7642450142450141,0.03790882473231565,0.5001455946443598,0.029188655009533084,0.4228176788373564,0.024780112799567985,0.00860396443668814,0.000624871983420895,0.33092969896178875,0.005380887353128073,0.8696677051904237,0.012341050186972945,0.4736913696419042,0.014947147093686546,0.9023434727049431,0.003921104397976069,0.1430440978881365,0.003509626506330469,0.4228109052199245,0.006154800168940804,,,,0.9273149090847037,0.021480963110959216,0.48644975844069477,0.03467122427212445,0.90172315641436,0.012529015981428812,0.1895832646966106,0.010898608745874642,0.47358596089561955,0.015797952804189035,0.475,9.446514923183713e-05,0.16219512195121952,,,,,,,,,,0.00018791118070078278,0.5192735229369373,0.00018791118070078278,Baseline
+13,0,Pref. Opt.,Phi,0.01,1,1,1e-05,0.9837146980004123,0.38233333333333336,0.04809841902243682,0.4695644471765722,0.032111022589819085,0.3821397462223834,0.02464703456072267,0.006153572657457546,0.0008310480099831638,0.33470161507723,0.0025907352255946717,0.7813390313390313,0.03683314429199754,0.49994997489122384,0.02901335848914468,0.423125714266884,0.024726769022537756,0.008729845972384517,0.00061717851022102,0.32820195660951695,0.004749790799072437,0.8699035708291118,0.012486878576912882,0.4723351180736019,0.014908368095607742,0.9022933428393556,0.003922594953766335,0.14342581530987908,0.0036072113448714724,0.42420604544307905,0.006353922574839655,,,,0.9329232942741275,0.021639744391332795,0.4871442826481921,0.03468910704930502,0.9008306477162641,0.012558588339405433,0.19130465211597325,0.01093519211074757,0.4693874139013009,0.015138469053562939,0.475,0.00010100967301696073,0.16341463414634147,,,,,,,,,,0.00018791118070078278,0.5178248457199529,0.00018791118070078278,Baseline
+14,0,Pref. Opt.,Phi,0.01,1,2,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.47373309033540345,0.03209292262222703,0.38306152117101555,0.024593148204354688,0.006221156974718491,0.0008326594451508989,0.33261632170052935,0.002403245145408713,0.7642450142450141,0.03790882473231565,0.5020158663025113,0.028922915356361156,0.42270417924164666,0.024547437598509053,0.008591716794152418,0.0006241901899955245,0.3250061811418768,0.0040736233139708,0.8697745042465942,0.012703071530420666,0.47070887156358365,0.014862566050645976,0.9016617458504347,0.0038924697723894657,0.1432510360204912,0.0036332228257412624,0.42169449673347664,0.006170992160783022,,,,0.9238480952795737,0.02270021192679422,0.49118586644200307,0.03439937641741466,0.8973157071423055,0.012643086433375313,0.18928482312551928,0.01128407740707521,0.4620826945233665,0.014870571758568919,0.475,0.0001079662160918815,0.16463414634146342,,,,,,,,,,0.00018791118070078278,0.5186540235064979,0.00018791118070078278,Baseline
+15,0,Pref. Opt.,Phi,0.01,1,3,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.47322356959353196,0.032090097799751256,0.3836252355966438,0.024533610249120985,0.006244834976411179,0.0008366171638060224,0.332023633957796,0.002240392424196764,0.7642450142450141,0.03790882473231565,0.5037126672984529,0.028674912968998895,0.42315810331987685,0.024509415202320455,0.008595310840958353,0.0006247365596962819,0.32273691517936487,0.004054460541639736,0.8671411814816133,0.012864763680041141,0.46965244104034704,0.014811643696071134,0.9011404530470776,0.0038774756771686758,0.14294274241861651,0.003652105525939985,0.42516070085930907,0.006360936337803339,,,,0.8882952136116742,0.030173871303531573,0.4941051785407263,0.034414952599980016,0.892011493086336,0.01269183107531916,0.181449464788547,0.011516363123766277,0.4692100338392125,0.015989732346313342,0.475,0.0001315457593591418,0.16829268292682928,,,,,,,,,,0.00018791118070078278,0.518710798875116,0.00018791118070078278,Baseline
+16,0,Pref. Opt.,Phi,0.01,1,4,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.4698419349800418,0.032174976991090856,0.3833710955901396,0.02450290256564993,0.0062216273555909555,0.0008326437977280568,0.32995903710748453,0.0025143808643199018,0.7642450142450141,0.03790882473231565,0.5043976451143791,0.02852554286309331,0.4225313101506524,0.02438108326503652,0.008606433771668401,0.0006252220539135593,0.3177757334605928,0.0040909182830694375,0.8659897604031571,0.01294696089031625,0.4679745149968511,0.01474244021831627,0.8997578733555137,0.003858208835410039,0.14277014790222292,0.0036595494875842414,0.42513923825054484,0.006558871646898568,,,,0.830626168311446,0.03666827582463525,0.500608886943749,0.034655168338932923,0.8803722774714562,0.01286604118769073,0.16720318733626194,0.011243049085207167,0.4832524940236804,0.0182485343806283,0.45,0.0001597403588675661,0.1719512195121951,,,,,,,,,,0.0005039436209702519,0.5178566010230325,0.0005039436209702519,Baseline
+17,0,Pref. Opt.,Phi,0.01,1,5,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.473392095236652,0.03213593743408392,0.38388315024452024,0.024406449715621222,0.006236562921041011,0.0008339162234818053,0.3262242787598863,0.002278723187388704,0.777065527065527,0.037358703521530014,0.5075911367243402,0.027999433867593158,0.4221688519543452,0.024079422131479598,0.008586217608143869,0.0005991179115282932,0.30961405773226625,0.0028007493631818773,0.8463808761428868,0.013662869153115336,0.4629802406039191,0.014643937089535087,0.8926058048504588,0.0038593252310868548,0.13866940284917637,0.0036620799876700303,0.4232339998206337,0.006775848826416689,,,,0.5951507612466873,0.04701737225270782,0.5147149210529406,0.03455717563377442,0.8146070831502457,0.014045559412524477,0.11782537863058522,0.010126005310337817,0.4539101489589914,0.01725294996501672,0.45,0.0003167852501064772,0.18536585365853658,,,,,,,,,,0.0005039436209702519,0.5176409900343251,0.0005039436209702519,Baseline
+18,0,Pref. Opt.,Phi,0.01,1,6,1e-05,0.9837146980004123,0.3973333333333333,0.04870415050172085,0.471955998541882,0.03202736283788483,0.38442990002238553,0.024287215038984594,0.0063868085765826765,0.0008364847580720083,0.32637109880998716,0.002559197570063446,0.7813390313390313,0.036833144291997535,0.5072079863087249,0.02789183907793472,0.422042399087405,0.024040452464352585,0.008606575802915802,0.0005947140631104189,0.31173464647688465,0.002725507451032846,0.8359647049487345,0.01398540865289182,0.46185472253185933,0.014589858185317082,0.8894214559114425,0.0038853990597116574,0.1370003718531207,0.0036825926266285296,0.42474502178168855,0.007106788904499209,,,,0.5952542795657114,0.046518730719322804,0.5170966115106268,0.034416950899108595,0.7897982301818713,0.014506903896328208,0.11572568933881214,0.009812552773005903,0.4549178026795063,0.0194299180535143,0.45,0.0004022172452096129,0.1902439024390244,,,,,,,,,,0.0005039436209702519,0.5189273306998731,0.0005039436209702519,Baseline
+19,0,Pref. Opt.,Phi,0.01,1,7,1e-05,0.9837146980004123,0.3973333333333333,0.04870415050172085,0.4735493554166035,0.03216060185135826,0.3843611343361813,0.024282058066674543,0.006373279533591777,0.0008344211040083778,0.3265799539701609,0.0025570664841415474,0.7813390313390313,0.036833144291997535,0.5072444156379285,0.02789171501484806,0.4209126843807497,0.0239404049635069,0.008602355342068378,0.0005935509605736122,0.3103928280800253,0.0027410198543817823,0.8353600028911543,0.014048605309541698,0.4615008801980296,0.014577311666534804,0.8876955822150964,0.00390206840606907,0.13688941799970575,0.0036891851650605874,0.41960781114981044,0.006886132345513996,,,,0.5511244707926591,0.04812972893552936,0.5217853217374565,0.03458208830203486,0.7776128715032737,0.014576091470149228,0.10842348927575078,0.010392964568508944,0.44939060451577123,0.019595271887938318,0.45,0.0005080382052256027,0.1951219512195122,,,,,,,,,,0.0005039436209702519,0.5187997231250767,0.0005039436209702519,Baseline
+20,0,Pref. Opt.,Phi,0.01,1,8,1e-05,0.9837146980004123,0.37733333333333335,0.04823455597984075,0.47283943704773734,0.032141106075115736,0.383321977821912,0.02418869749218474,0.006103657996804235,0.0008345726825392287,0.3269799150563427,0.0028114831645313235,0.7813390313390313,0.036833144291997535,0.5067470299325614,0.027740354553480756,0.4202048621937792,0.02386389710936895,0.008603423271328028,0.0005944545817256467,0.3094905798253412,0.002639950983222656,0.8344948145791741,0.01403317147400795,0.4604241035389309,0.014568576185452892,0.886103321944917,0.00389572569488024,0.1362255766893378,0.003624717948355866,0.42320804701124765,0.006997757958268458,,,,0.5557046276624373,0.04885660018778633,0.5249199913902439,0.03484705804801916,0.765206755396493,0.014758164767542933,0.11129006545878325,0.011225475130762672,0.4572348155118309,0.020601901219586695,0.45,0.0005698828390450217,0.1975609756097561,,,,,,,,,,0.0005039436209702519,0.5141204212831718,0.0005039436209702519,Baseline
+21,0,Pref. Opt.,Phi,0.01,1,9,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.47305201246994344,0.032025513070615026,0.3832585282122411,0.024153828510293955,0.006304204898528629,0.0008434781351910319,0.3272734485304209,0.002393918575076332,0.7856125356125355,0.03679964654194333,0.5076125619284477,0.0276422982768888,0.42000406726644474,0.02383728784491541,0.008661175169087286,0.0005957154226671271,0.31382433780907093,0.003291634016670384,0.8254588376729868,0.014312619333965044,0.45980541241197526,0.014559037103665752,0.8848184025706329,0.0039110395550623085,0.13498759255467818,0.003674858996123534,0.42578183612723514,0.007072260177702463,,,,0.477951365296287,0.04745327731911095,0.524225102206236,0.034466321020753574,0.7582780629722929,0.014833343353199312,0.09941397764465934,0.012266559601245825,0.4605282818832631,0.021643835207248004,0.45,0.0006032901037542615,0.19878048780487806,,,,,,,,,,0.0005039436209702519,0.5159019141261182,0.0005039436209702519,Baseline
+22,0,Pref. Opt.,Phi,0.01,1,10,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.4718799285103602,0.03207022632395998,0.3828481651107484,0.02416978696334627,0.006304494678789557,0.000843004449906009,0.3248681767383259,0.002600804822575272,0.7813390313390313,0.036833144291997535,0.5084242173163721,0.02770253830261805,0.4204093305477784,0.023855312368164842,0.008593357612077069,0.0005937218283935182,0.3092155113623794,0.0027477579524202095,0.8200642374948647,0.014478779490283391,0.45955791143341695,0.014534720156410472,0.8842474989015223,0.0039007334447722845,0.1338469232637897,0.003659254550298215,0.42424670062669245,0.0069936795561650784,,,,0.47666320019540215,0.04835504135185417,0.5258763051003069,0.03454144151614293,0.7540888513742684,0.01480332725423416,0.09900630246176083,0.012275419712231391,0.4457449478483344,0.020271071184639517,0.45,0.0006032901037542615,0.19878048780487806,,,,,,,,,,0.0005039436209702519,0.515327672412302,0.0005039436209702519,Baseline
+23,0,Pref. Opt.,Phi,0.01,1,11,1e-05,0.9837146980004123,0.38733333333333336,0.04848034069953959,0.47224980204856204,0.032126325825416184,0.38331229424569524,0.02416273620920019,0.006315111568157691,0.0008448801605183565,0.32522094906958254,0.0025543634145486858,0.777065527065527,0.03686234087422618,0.5069446282771966,0.027699732362282264,0.4199203215179363,0.023816242867843656,0.008521695353501987,0.0005903308922614756,0.3097998659013392,0.0026540286459362616,0.8234366808895417,0.014403973858602778,0.4601889564942228,0.014500234637120713,0.8841265051179884,0.003911412682269276,0.13434480838369153,0.003641417434490408,0.42186528079126506,0.006919430156173156,,,,0.477355072986262,0.04724224268622652,0.5250077492402552,0.0342066641899655,0.752332011401004,0.014861161664184788,0.09868799422318655,0.011459322053894659,0.4785914891769881,0.02371184522102981,0.425,0.0006384575062838849,0.2,,,,,,,,,,0.0012708143485281624,0.5152420182060062,0.0012708143485281624,Baseline
+48,0,Retain Model,Phi,0.01,1,-1,2e-05,0.0,0.43733333333333335,0.04938517614523233,0.4566326936712402,0.03302099494178741,0.3808582408768965,0.019657218278108554,0.00693177988902883,0.000837165528065107,0.33673456869232615,0.002895923239621764,0.7215099715099714,0.04019992314443127,0.49079303584497147,0.029332542237957754,0.4091986973040716,0.01813497686131319,0.008160041024810489,0.0006259427718389491,0.33083919148372004,0.004875766466783252,0.8821586065505498,0.011300871600387114,0.47814597044432,0.01470878267456817,0.9055798983042737,0.0036737327308074127,0.14686273673169498,0.003552445628334266,0.45246705756499705,0.007829978128825415,,,,0.4320950028850645,0.02596101749250188,0.6576937980932389,0.034791136763264396,0.17052496661467564,0.02992203970918228,0.0815255064751508,0.004202778891891995,0.45844616301248403,0.018226628388955366,0.0,1.0,0.0,,,,,,,,,,1.0,0.5202691368576465,1.0,Baseline

versions/phi-5p/phi-5p.csv

@@ -0,0 +1,45 @@
+,index,Method,Model,WD,Forget Rate,Epoch,LR,Compute,ROUGE Real Authors,ROUGE SEM Real Authors,Truth Ratio Real Authors,Truth Ratio SEM Real Authors,Prob. Real Authors,Prob. SEM Real Authors,ROUGE-P Real Authors,ROUGE-P SEM Real Authors,TTR Real Authors,TTR SEM Real Authors,ROUGE Real World,ROUGE SEM Real World,Truth Ratio Real World,Truth Ratio SEM Real World,Prob. Real World,Prob. SEM Real World,ROUGE-P Real World,ROUGE-P SEM Real World,TTR Real World,TTR SEM Real World,ROUGE Retain,ROUGE SEM Retain,Truth Ratio Retain,Truth Ratio SEM Retain,Prob. Retain,Prob. SEM Retain,ROUGE-P Retain,ROUGE-P SEM Retain,TTR Retain,TTR SEM Retain,KS Test Retain,Wilcoxon PVal Retain,Wilcoxon Stat Retain,ROUGE Forget,ROUGE SEM Forget,Truth Ratio Forget,Truth Ratio SEM Forget,Prob. Forget,Prob. SEM Forget,ROUGE-P Forget,ROUGE-P SEM Forget,TTR Forget,TTR SEM Forget,KS Test Forget,Wilcoxon PVal Forget,Wilcoxon Stat Forget,KS Test Real Authors,KS Test PVal Real Authors,Wilcoxon PVal Real Authors,Wilcoxon Stat Real Authors,KS Test Real World,KS Test PVal Real World,Wilcoxon PVal Real World,Wilcoxon Stat Real World,KS Test PVal Retain,KS Test PVal Forget,Model Utility,Forget Quality,Submitted By
+126,0,Finetune Model,Phi,0.0,0,-1,2e-05,0.0,0.41733333333333333,0.04908701817726713,0.4476925527525571,0.03227110957959653,0.3716867131781305,0.0189665917514988,0.006609210896948858,0.0008323188534979413,0.32849747821093095,0.0025638903409710874,0.7556980056980056,0.03841079236981315,0.4960620278903896,0.02911364407012106,0.40949566673960025,0.018342034313532797,0.008506353470348813,0.0006169734981950858,0.32392206897329384,0.005074759040830575,0.8681882164333115,0.012586893544718975,0.4768480193904878,0.01476904074691066,0.9024875417830132,0.003933498315197806,0.14292843594668517,0.003563553395870209,0.41340711817551534,0.005926892433531737,,,,0.8752167884046598,0.011809089858149302,0.4882534392910444,0.013534006024118014,0.9086918278343834,0.003939262001123021,0.17179131104641562,0.004405566476237556,0.4299893732259693,0.005748680600806638,0.3566666666666667,4.665567748131161e-44,0.0363233665559247,,,,,,,,,,2.4311282147882553e-17,0.515509929080137,2.4311282147882553e-17,Baseline
+67,0,Grad. Ascent,Phi,0.0,5,0,1e-05,0.9473684210526315,0.3973333333333333,0.04870415050172085,0.44684106106890525,0.032137622250196035,0.3712373853602735,0.018938009179897658,0.00632999116673955,0.0008299128014062314,0.3286833099762514,0.002696334094594899,0.7745014245014245,0.037116781834202695,0.49452339896487557,0.029243331793233778,0.4090005324349409,0.018292400348495445,0.008432736421731887,0.0005608880420400836,0.3202792516336548,0.004085016694472165,0.8581532250665725,0.012868980941485218,0.47582046068048717,0.014778784826820228,0.8988625297962449,0.004103236213125108,0.1410789764587879,0.0035611437309264475,0.4062220907437317,0.005507264693461038,,,,0.8404284586395824,0.01566784994090144,0.47858954725712105,0.016183039427127104,0.8742973307336706,0.007143208141194536,0.1636152721853565,0.004975550444156972,0.4216835871657325,0.005729777202450207,0.395,1.4125429096205823e-27,0.056318407960199005,,,,,,,,,,2.5656301025789152e-14,0.5117667972582467,2.5656301025789152e-14,Baseline
+68,0,Grad. Ascent,Phi,0.0,5,1,1e-05,0.9473684210526315,0.33233333333333337,0.04657628332477988,0.44210118329672776,0.032165824322996486,0.370096462479143,0.019282498227176433,0.005425422031291212,0.0008144717272438095,0.32929128240797945,0.003008976304023689,0.7457264957264957,0.03851880335961918,0.492831479884917,0.02917419633521638,0.40881121801319376,0.01845354215645646,0.00822505633941246,0.000601149907125342,0.31801362058145605,0.004008271347472442,0.6590670847004928,0.014677100366852339,0.4721086935306992,0.014807506644332207,0.8000129691423457,0.007332337835836101,0.10572096622675228,0.00302914844107105,0.3829866347521733,0.0038455623793949164,,,,0.523875512629707,0.015129502017233105,0.486929233286044,0.01594601323690575,0.5538022122973938,0.01691250745728419,0.09782466068862479,0.0031246978706693266,0.4032616523246594,0.004696836452816383,0.38,2.6290961095202044e-24,0.08522388059701493,,,,,,,,,,2.9623487356936016e-13,0.4819389253922607,2.9623487356936016e-13,Baseline
+69,0,Grad. Ascent,Phi,0.0,5,2,1e-05,0.9473684210526316,0.022000000000000002,0.012274635093014647,0.39620073064591255,0.03250722397267746,0.34496722174968525,0.019515997750426256,0.0004174184166420191,0.0002358789725854864,0.42848145602999893,0.013578741674543676,0.37279202279202284,0.041323420793097684,0.442190376311666,0.029895924891259185,0.3721230910785776,0.018138497625154938,0.004228818041993495,0.0005025942131026081,0.341141540426054,0.009009176175940025,0.3547117195461819,0.008477417921476105,0.3842483779626528,0.015768395507761424,0.14814497753894018,0.007766095263683715,0.055387973752890916,0.0012171717868512183,0.44077760101187835,0.007224482512004959,,,,0.3437007773493141,0.010875728034148102,0.572027210489125,0.016800512973190972,0.07475984369847262,0.007714203065864513,0.06587151313911072,0.002491038357300761,0.4371590132129096,0.008668335795261717,0.205,1.3925915431369171e-05,0.3228358208955224,,,,,,,,,,0.0004306111484986726,0.12733378717798044,0.0004306111484986726,Baseline
+70,0,Grad. Ascent,Phi,0.0,5,3,1e-05,0.9473684210526315,0.002,0.001999999999999999,0.3099229279405089,0.036776766757030475,0.25015443587005765,0.0237599691843799,7.142857142857142e-05,7.142857142857143e-05,0.23814117160515258,0.0036164544179831267,0.023076923076923078,0.008304661733060714,0.3649725522142139,0.030986425937952835,0.29460670979135184,0.01846116477745002,0.0004957599412592635,0.00016995877026139934,0.23192193481705609,0.00300847483104681,0.1328676256480963,0.0037889158585325835,0.19641867038930472,0.013997180474537246,5.478569225841231e-08,4.868139882026174e-08,0.022577162493676983,0.0007259684937902722,0.24385593081901352,0.002516376223338923,,,,0.1315962842301884,0.004454278661259192,0.7088498260680648,0.0151441636887798,9.176661894656182e-10,6.516838609006018e-10,0.026948172267243855,0.0010313984446637737,0.24809034478846903,0.0027618824671781163,0.225,3.3454595325018634e-05,0.3308457711442786,,,,,,,,,,7.536785951251345e-05,4.930558519139861e-07,7.536785951251345e-05,Baseline
+71,0,Grad. Ascent,Phi,0.0,5,4,1e-05,0.9473684210526316,0.002,0.001999999999999999,0.41833301411084833,0.04331888426812115,0.25175750401052766,0.03261552866485664,7.936507936507937e-05,7.936507936507938e-05,0.2479745360712376,0.003949256422913901,0.028774928774928776,0.009099120002652528,0.38804860446753614,0.03686918277107514,0.24595058285050908,0.019695508938273522,0.0006693115658605177,0.00020354599553303266,0.25072748375852244,0.004242559965108977,0.1370976168462799,0.0038052310706161185,0.12240294101298028,0.00976717941844421,1.3697292989730826e-23,4.06196238277166e-24,0.025209902530529328,0.0007335856981830234,0.24606743075889081,0.0025727085763113926,,,,0.14001168507329345,0.003990527452962912,0.7818461264805139,0.01186528822868957,9.593948969309419e-24,4.736289036655825e-24,0.03127298577067527,0.0010949389111112313,0.24591376890305824,0.0031751463019621356,0.325,1.4058450645425207e-11,0.2244776119402985,,,,,,,,,,9.594235055800515e-10,1.2327563690757743e-22,9.594235055800515e-10,Baseline
+47,0,Grad. Diff.,Phi,0.0,5,0,1e-05,0.9151450053705693,0.4073333333333333,0.04890628707870714,0.4466159282714341,0.03227725675703351,0.3710734648261002,0.018969486771700787,0.006468110761036569,0.0008314572173874184,0.3304981854878406,0.0025720107545074396,0.7727920227920227,0.03738327846205438,0.4949143219816758,0.029248827596421747,0.4084623408686101,0.018313146487266986,0.008613470615930808,0.0006070570260083769,0.3249173972853186,0.005032225533961778,0.8679063293607155,0.012509475871902834,0.47705404916738653,0.014746875614097543,0.9026283843948719,0.0039192180377366,0.14350239653535404,0.003619238084366768,0.4122675076781909,0.0058937700821408015,,,,0.8554831372014604,0.015156852623678194,0.4788210584027038,0.016205041167035833,0.8841020451804712,0.006635012194431027,0.1671629305729499,0.005046930533975713,0.4239164739708372,0.006360035984273378,0.39,1.5512613290165279e-27,0.056666666666666664,,,,,,,,,,5.86730162802883e-14,0.5140514356486094,5.86730162802883e-14,Baseline
+48,0,Grad. Diff.,Phi,0.0,5,1,1e-05,0.9151450053705693,0.354,0.04780896327778893,0.4474506637994592,0.032153797204988344,0.3700535221772105,0.019016517672948714,0.005627922094434177,0.0008149540094834378,0.3347690644626086,0.0029042198696014455,0.7445868945868945,0.03816096397699633,0.49308992058372053,0.029313095537361167,0.407060128056712,0.018468360106436304,0.008017593756163992,0.0005607013996057553,0.3266808597255474,0.004038621034504087,0.8171762691461101,0.013942692697798669,0.4769894140137838,0.01481242305225521,0.8849544905515874,0.004732991293692389,0.1346270094380487,0.0035411083254428526,0.4206811837858982,0.005992512078368245,,,,0.6357862300795125,0.016608236689450863,0.48562036057335317,0.01634862804989653,0.6922566538424771,0.014901061670204682,0.12174278946222708,0.004122726043901429,0.432357181478138,0.006786617393036778,0.37,6.039345239174551e-26,0.07049751243781094,,,,,,,,,,1.4275699621532978e-12,0.49889251435971,1.4275699621532978e-12,Baseline
+49,0,Grad. Diff.,Phi,0.0,5,2,1e-05,0.9151450053705693,0.21033333333333332,0.03889522314935838,0.4458044657180148,0.03291473744174133,0.3706000268678954,0.019968170680658787,0.0032874515464930837,0.0006387295095144267,0.3636776929727203,0.0069209558379519035,0.6377492877492877,0.04191931124791538,0.4866600306735924,0.02931634917484688,0.4030100114413805,0.01883084176589643,0.006899638275322906,0.0005655825385103126,0.33969942562405747,0.0060591539918628656,0.5365059942109692,0.012840808316645441,0.44940451904946305,0.015461048074510955,0.6226823417731935,0.01027240182004234,0.08362670221646601,0.0020308381893048262,0.42493705332236287,0.005622893295803352,,,,0.42430797264120346,0.012027947107023334,0.5192282079290663,0.016578048414717106,0.2556903609012649,0.014207422319297013,0.07950300535186457,0.0021961532717120306,0.46013561601999037,0.007248692979126342,0.305,7.526258864342888e-16,0.17129353233830846,,,,,,,,,,1.2940880677508261e-08,0.41870163807342897,1.2940880677508261e-08,Baseline
+50,0,Grad. Diff.,Phi,0.0,5,3,1e-05,0.9151450053705693,0.18866666666666668,0.037876342008575974,0.44080970247976387,0.03385663092526476,0.3682838234001564,0.02060090292052789,0.0031120606294756763,0.0006546514694008205,0.3617080283539992,0.007777708065010618,0.6064102564102565,0.04350608581912611,0.4834544538651108,0.029425157576296296,0.3988631057055157,0.019052938183065072,0.006890961271525765,0.0006193691139822861,0.32645971443860383,0.006770371749589383,0.49370522130160877,0.012264526133580611,0.4454011022794278,0.015835719187281088,0.543184071685971,0.01152658002194726,0.07943072320831535,0.002071543274226305,0.44170676211618826,0.00713748465134117,,,,0.364180708737891,0.010817582072742232,0.5580607157880576,0.01623976447666189,0.08540142340110052,0.008637568740666405,0.07231435115420254,0.002628328546264106,0.4627349431439022,0.0087320847149056,0.205,1.5688404677317863e-07,0.28616915422885575,,,,,,,,,,0.0004306111484986726,0.3978693876024443,0.0004306111484986726,Baseline
+51,0,Grad. Diff.,Phi,0.0,5,4,1e-05,0.9151450053705693,0.122,0.03186397858574995,0.44201943800402055,0.035759856692931226,0.3683926428653211,0.022475206467387595,0.0021325904810737505,0.0005845336119032885,0.4186515159601691,0.01231097099943678,0.577920227920228,0.04364979486736906,0.4715714472208501,0.030808753346686125,0.3922489991013043,0.019659774564171537,0.00620346174398396,0.000557828282165281,0.33494549709956234,0.009048567220464836,0.4395359423938254,0.011224843985016032,0.4418577697662992,0.01656713947699931,0.4255116573486394,0.012196601454876147,0.07356505022571179,0.002412492660724484,0.46380544987009537,0.008476839143756867,,,,0.29646604270404203,0.009310816708077962,0.5904378175819995,0.016469076073843042,0.020702010845669888,0.003437824405133252,0.06287302618362683,0.0036584358032777375,0.4920935418592645,0.010842643256210854,0.125,0.046045252201116084,0.41865671641791047,,,,,,,,,,0.0878382826713132,0.3401066999915881,0.0878382826713132,Baseline
+77,0,KL Min.,Phi,0.0,5,0,1e-05,0.8678839957035446,0.3973333333333333,0.04870415050172085,0.44663743388121696,0.032220977795081776,0.3716177188089344,0.018959206554569223,0.006327701629451819,0.0008299419543136753,0.32951770011354387,0.0028592280712262876,0.7599715099715099,0.03792058995790417,0.49527423191494396,0.029155572922248844,0.409186375813588,0.018302215006078114,0.008482696639498691,0.0006098415518577068,0.3260959147825401,0.00498795111941358,0.8528023851574646,0.01311582130764109,0.4761049625929941,0.01478958175558384,0.8990750725714205,0.004093857241809358,0.13999067753149338,0.0035375629904701705,0.4054830823958956,0.005551602308010087,,,,0.8432135702347838,0.01580486992729008,0.4786781129652564,0.016164646539066536,0.8743641304258404,0.0071443523726619045,0.16420552950303727,0.005041787982213399,0.42460929861287966,0.006078788250462324,0.395,1.797017872812512e-27,0.05721393034825871,,,,,,,,,,2.5656301025789152e-14,0.511052918910662,2.5656301025789152e-14,Baseline
+78,0,KL Min.,Phi,0.0,5,1,1e-05,0.8678839957035446,0.3423333333333333,0.0469277329647632,0.4414456943629027,0.032063243657108643,0.36925920995403844,0.019224369999395443,0.005556134982935437,0.0008162867731712584,0.32864062437794694,0.002848095882670432,0.7457264957264957,0.03851880335961918,0.49168265548049883,0.029211968468783368,0.4071703986814626,0.018441352288409756,0.008219221377125484,0.0006011107893735244,0.3205347461535208,0.004036913377075627,0.6724316199735322,0.01488015542370608,0.47277090117945847,0.014803345218428188,0.8072019612131972,0.0071765983826697945,0.10794732473179446,0.0031135166539973964,0.389509351207129,0.004355308590792392,,,,0.5332292670299291,0.015416733853042255,0.4871582621320234,0.016011430415461495,0.5601068250267155,0.016902455039152265,0.09943528162795973,0.003105289145186037,0.4070759438764393,0.004610021250081802,0.38,1.781581884339874e-24,0.083681592039801,,,,,,,,,,2.9623487356936016e-13,0.48474413452709203,2.9623487356936016e-13,Baseline
+79,0,KL Min.,Phi,0.0,5,2,1e-05,0.8678839957035446,0.04366666666666666,0.018316919496543987,0.39806635712070365,0.032843959014959734,0.3464502539584891,0.01973515449250203,0.000898944305674319,0.00037398524238592596,0.44046717217608533,0.013373456043173393,0.3784900284900285,0.04149565052966719,0.4458137265709391,0.029812147990470866,0.3755313993938293,0.0182840533348504,0.004278453268342547,0.0004955301931897874,0.35759431378091616,0.00956450761156174,0.37338848968149757,0.008287734257554162,0.3942435705776056,0.01581732788474261,0.18450891569750222,0.008517619084867836,0.05849733028459261,0.0011782596011925203,0.46441445252807484,0.007298071939226236,,,,0.35319707533003564,0.010539359290917764,0.5626663464843066,0.016603109035684408,0.08730251764724674,0.008654525939224302,0.06611500199220717,0.0019985879055218246,0.4690523957367925,0.008159883601725175,0.205,7.842405822239322e-07,0.2986069651741294,,,,,,,,,,0.0004306111484986726,0.19362393928050475,0.0004306111484986726,Baseline
+80,0,KL Min.,Phi,0.0,5,3,1e-05,0.8678839957035446,0.002,0.001999999999999999,0.30201302727581086,0.0359292428310432,0.26361026590493186,0.023669586935487867,7.042253521126761e-05,7.042253521126761e-05,0.37208680570375824,0.011372333282637365,0.025925925925925925,0.00871496805639447,0.34315526944088404,0.030043262003420727,0.2993239828270343,0.01761427595122048,0.0005446088362680811,0.0001753009382007315,0.32755571582894194,0.009274441272395262,0.16462274937694432,0.0037441881048317577,0.23340822944223796,0.015594749879293855,4.173823224567624e-06,3.3923915407076885e-06,0.02863948585676244,0.0008235456291004619,0.4142926864611494,0.005046267352281318,,,,0.16755387682346515,0.0043857243965299665,0.6757735421664998,0.016853723554608385,1.220522886156559e-07,9.367977233877392e-08,0.034043790301799555,0.0011484185321259004,0.44339933104703827,0.006272589700842167,0.16,0.005672664607292997,0.3872139303482587,,,,,,,,,,0.011843449760085422,3.747645558599556e-05,0.011843449760085422,Baseline
+81,0,KL Min.,Phi,0.0,5,4,1e-05,0.8678839957035446,0.002,0.001999999999999999,0.4097554436714871,0.043210779822642806,0.25275127439977985,0.03190887177655261,7.874015748031496e-05,7.874015748031494e-05,0.28378025704540655,0.008591776274083132,0.030911680911680908,0.009772908917887905,0.39049857744691524,0.03617729568496359,0.25029845172286735,0.01945037923292546,0.0006886041041805987,0.00021678746067521545,0.30303401011287995,0.007954675844391936,0.14568521435612258,0.004252948004178069,0.12414692796642322,0.009818249002041155,2.317394574739555e-22,8.629747369359599e-23,0.026308783985412032,0.0007884050133616958,0.252999881825323,0.004199019656606275,,,,0.14998421238846216,0.00447613477254901,0.7787039479377412,0.011911244694310218,1.1717383707761457e-22,5.573994496683587e-23,0.03270778392003661,0.0011920840631060659,0.26105827234348133,0.005975724171254671,0.325,1.4417929326611711e-11,0.22462686567164178,,,,,,,,,,9.594235055800515e-10,2.0856551172655995e-21,9.594235055800515e-10,Baseline
+57,0,Pref. Opt.,Phi,0.0,5,0,1e-05,0.9151450053705693,0.4073333333333333,0.04890628707870714,0.44810881062315006,0.03230929237527726,0.37302225807484596,0.018907860573119967,0.006497527275954093,0.0008345604957210465,0.3225948185196416,0.0023869332707901567,0.7642450142450141,0.03790882473231565,0.4987951852969569,0.028760220187570445,0.41024356885819757,0.01805787238521131,0.00855172227063477,0.0006139222913576342,0.31205095443832886,0.002655900862762239,0.8643778629043591,0.012985548917791818,0.47240205238314575,0.014659213817991427,0.9010948069213501,0.003894524752710786,0.14221399867577636,0.003610946772901936,0.4123466668509634,0.006198708213044482,,,,0.8782605380348965,0.014487186400751364,0.48170693655177294,0.016049116085947612,0.899109665612838,0.005577463156717353,0.1726962885310151,0.0051875264428679815,0.42117066324841923,0.006383195814722743,0.38,4.781022731812138e-26,0.06960199004975125,,,,,,,,,,2.9623487356936016e-13,0.5142331318824949,2.9623487356936016e-13,Baseline
+58,0,Pref. Opt.,Phi,0.0,5,1,1e-05,0.9151450053705693,0.2906666666666667,0.044871883967795526,0.4180911036532261,0.03183262536887489,0.3594850397844364,0.017651168910767602,0.0048148711119298335,0.0007877507722935037,0.2958753887999601,0.004447523395247747,0.6951566951566951,0.041919636744596506,0.5040280769699952,0.026982782604721034,0.40925633776426124,0.017015823108356493,0.007591325302186247,0.0006132694133633984,0.3128593063326545,0.005826898535698281,0.40718715267578204,0.019256985236190162,0.44548365214328245,0.014056675287694364,0.7788888643061213,0.00579210738401997,0.06358113984104231,0.003189694674669916,0.3894178518888129,0.009307428345098675,,,,0.2625803997123761,0.015673006617937347,0.5163360891975817,0.01588842538010633,0.7343346613755071,0.00761851481285614,0.05194716964122177,0.0032953101948732964,0.4189208367730519,0.012018512931149335,0.325,4.865965117919374e-17,0.15791044776119403,,,,,,,,,,9.594235055800515e-10,0.43978706341328533,9.594235055800515e-10,Baseline
+59,0,Pref. Opt.,Phi,0.0,5,2,1e-05,0.9151450053705693,0.16733333333333333,0.03688785443359108,0.40964080503277467,0.031160081876296784,0.3539553060213049,0.017210033824156628,0.002714483887411253,0.0006350452078196279,0.28844425727597955,0.00852304032469176,0.6467236467236467,0.043776404050617786,0.4979123607425766,0.027039099712868407,0.40719847158387357,0.016708866717683832,0.007116150775399226,0.0006227169514825036,0.2962724455911795,0.007695681348683412,0.3186203114165437,0.016873102470429335,0.44092265206594294,0.013820253376178884,0.75688583654713,0.005994524614368157,0.05146068885227744,0.0029385573652789486,0.408228547768255,0.01070626270308944,,,,0.20018507917968476,0.009660411061174604,0.5247844608001638,0.015624189946729863,0.6730846169333102,0.007636603890921891,0.04013887919022475,0.0024361138045957504,0.4330279620607652,0.012805324056853824,0.315,6.406899025072769e-15,0.18213930348258706,,,,,,,,,,3.602472544567432e-09,0.3753029530277975,3.602472544567432e-09,Baseline
+60,0,Pref. Opt.,Phi,0.0,5,3,1e-05,0.9151450053705693,0.28733333333333333,0.044963584217941346,0.41718141307240947,0.03158033560399835,0.3582971059371289,0.01761460068210511,0.004734439304363033,0.0007827926425963448,0.2934308534866858,0.005154214445855657,0.7051282051282052,0.04172849403345307,0.498369216561253,0.0274783977707353,0.41076561169103987,0.016946986041210737,0.0075980437625121225,0.0006164580059362407,0.3002300088075271,0.006256128850514236,0.5477980797715342,0.020693281068159458,0.4497794911496921,0.013830244476730216,0.8064628470576224,0.005212395609476496,0.08640832917309997,0.003706449606951602,0.35298611862812906,0.0067437229061464475,,,,0.19946583988927535,0.01075092014330646,0.5179552804716727,0.015483008846305001,0.7115012654055353,0.007434179379594254,0.0397281262204839,0.0026196094391503027,0.3147341160834003,0.008490841603606216,0.325,2.3645183456640836e-16,0.16557213930348258,,,,,,,,,,9.594235055800515e-10,0.4541718815332515,9.594235055800515e-10,Baseline
+61,0,Pref. Opt.,Phi,0.0,5,4,1e-05,0.9151450053705693,0.289,0.045011670764930574,0.4197342147868375,0.03169685066852125,0.3594715092966317,0.017831669358283044,0.0046467806845421405,0.0007730581842911,0.2943513595115394,0.0033432857060786735,0.7421652421652422,0.039980660506368816,0.4942782498998667,0.02798105690756216,0.41057631724515115,0.017227815023243147,0.007892750629459037,0.000602423007071051,0.2956308366194299,0.005280440358280402,0.6431082659335433,0.01968654713197535,0.45723983714878896,0.013955300895140235,0.8307311094580224,0.005001956848352048,0.10272980502028747,0.0038340232630478697,0.3610818051070236,0.005838759754859062,,,,0.2142412330267501,0.0131927757896015,0.5092541998330259,0.01551136757395992,0.7337058652111381,0.007348494338207295,0.041371837695604,0.002756106904444783,0.2956168969069584,0.0052211321228071445,0.34,4.0331927814505295e-18,0.1461691542288557,,,,,,,,,,1.2127544312107394e-10,0.46448175226712884,1.2127544312107394e-10,Baseline
+83,0,Retain Model,Phi,0.0,5,-1,2e-05,0.0,0.4181666666666667,0.048417780054074745,0.452619425551523,0.03276880148869164,0.37672239597055446,0.01997551803220911,0.006770338014656243,0.0008415059163335462,0.38559041457237286,0.004703268905875031,0.7542735042735043,0.03803758218098353,0.5000344040307756,0.029891640628495534,0.41581775420489836,0.01853256215814981,0.008496280770998807,0.0006094230662782937,0.3637387836358756,0.005406283360404024,0.874148114684597,0.011704179986916521,0.47604676909766463,0.014778282281734924,0.9032318155028709,0.0036879515961418794,0.1477059540177595,0.0036282419998955413,0.5013622204205103,0.006937446008008315,,,,0.42377558329524495,0.011929022984044548,0.6304729729096896,0.016015199183240493,0.13919658565290485,0.010124524528174313,0.07861874997257551,0.0018628484602644802,0.5097503976815042,0.008120088868810355,0.0,1.0,0.0,,,,,,,,,,1.0,0.519105725724611,1.0,Baseline
+50,0,Retain Model,Phi,0.01,5,-1,2e-05,0.0,0.42483333333333334,0.04875756869502934,0.45316206859199293,0.03264141366244881,0.37652852177657875,0.019938203231217873,0.006830683283714753,0.0008441486901540186,0.38193041689621005,0.00508030085542245,0.7514245014245015,0.03841420788700116,0.4996699066372895,0.02988294175220104,0.415358088017408,0.01856589722762314,0.008419340748676295,0.000614840909663249,0.3716823986479536,0.005860168184119516,0.8730404742518771,0.011887122608792737,0.47646922056672225,0.014784468854096159,0.9029671577901152,0.0037056191445714475,0.14748570805725,0.003641886728964624,0.49837190765545397,0.007436411553182218,,,,0.4287828600171711,0.011889233433531439,0.6308750426217175,0.016057591005249512,0.13932983163434137,0.010164844693333376,0.0794925659968092,0.001858261239460288,0.5170377343029624,0.008508323382192614,0.0,1.0,0.0,,,,,,,,,,1.0,0.5199978641100643,1.0,Baseline
+62,0,Grad. Ascent,Phi,0.01,5,0,1e-05,0.9473684210526315,0.38733333333333336,0.04848034069953959,0.4702205317221988,0.032006711055923516,0.38116377863356116,0.024657890803697315,0.006236020129651401,0.00083416453982487,0.33305425244477094,0.002522676081370665,0.7727920227920227,0.03738327846205438,0.5008360231562771,0.029131041282461387,0.42339562816497206,0.024822738104842076,0.00865043176576349,0.0006188786958300367,0.3262319675068413,0.004169588336868625,0.8630342336335792,0.012508602742289104,0.47230665884023193,0.014940931399018519,0.8991764596830294,0.00408061058549671,0.14224332865284828,0.0035741055630574494,0.4207631827652003,0.006075643818175603,,,,0.8411565288268551,0.015898061481940798,0.48139415398737057,0.016236296229064178,0.8740048697666535,0.0071617181689399055,0.16368861698397716,0.004983194993253389,0.4390333927172609,0.006967717879264403,0.395,5.0657070725933796e-12,0.21850746268656718,,,,,,,,,,2.5656301025789152e-14,0.518057494086161,2.5656301025789152e-14,Baseline
+63,0,Grad. Ascent,Phi,0.01,5,1,1e-05,0.9473684210526315,0.33233333333333337,0.04657628332477988,0.46352625928363816,0.03220260155742308,0.378287678659287,0.02505813605076499,0.005438293103668776,0.0008160732123530282,0.33399058971499124,0.0025099941811678747,0.7670940170940171,0.03702451857933602,0.49877845953370964,0.02920750939548373,0.4269426301532781,0.02548451127551873,0.008428108495527019,0.0006018951023284206,0.3174084103370284,0.0028508944459367212,0.6994548680944075,0.014956253252312483,0.46741417161467724,0.015016271667911646,0.8207478715733544,0.006813977262923422,0.11231962010095738,0.0031904904996429523,0.4009420199109086,0.005211912879279698,,,,0.5427657287778972,0.015772823413817087,0.48859216878703704,0.016100075552418328,0.5910403224955716,0.01651648379973555,0.10126851053486309,0.003283346105068612,0.41594824903789485,0.00545838878356855,0.385,6.455513252234515e-11,0.2336318407960199,,,,,,,,,,1.3261055712130201e-13,0.49320032068312797,1.3261055712130201e-13,Baseline
+64,0,Grad. Ascent,Phi,0.01,5,2,1e-05,0.9473684210526316,0.15283333333333332,0.03500044492261698,0.44489944075465077,0.03367259136018443,0.3679378231916626,0.02709062037048553,0.0023504843259260456,0.0005476038130725186,0.3738117274932104,0.009328626906495371,0.5180911680911681,0.043372146202152476,0.46774111236523674,0.02920334262448001,0.4194455560942563,0.02797537387604843,0.005689552572439171,0.0005732423200484731,0.3288924817165697,0.005317839481290448,0.407576475324819,0.009485677754656576,0.41575931813309824,0.01557671659740291,0.31023717642079096,0.010414590938182467,0.06460684160231213,0.0024076691622645975,0.39836579562816393,0.0055500024222479485,,,,0.39561813737893253,0.011014054482506298,0.528836311671055,0.01609422528076213,0.15419687807693006,0.011550388063867842,0.07365621288704191,0.002177349407547025,0.4240794974031419,0.006728028581349522,0.27,5.872102308255953e-07,0.296318407960199,,,,,,,,,,8.056669472003223e-07,0.34558327906881525,8.056669472003223e-07,Baseline
+65,0,Grad. Ascent,Phi,0.01,5,3,1e-05,0.9473684210526315,0.013666666666666667,0.007060663863002368,0.38499512052448176,0.031869189691370994,0.3285657729627356,0.02569241479592782,0.00027888115019786963,0.00013734550421161686,0.45302379910702145,0.014960818903311992,0.1854700854700855,0.03192740576456628,0.4129756012061883,0.030697548285939392,0.39253027406360197,0.028576237195492765,0.0024177315696571,0.00042124508488465567,0.40204239431066974,0.013992213046465115,0.27498685935440026,0.008109027780830868,0.3463962260149917,0.016249747008218216,0.035639351532210393,0.004070251115426569,0.0435541376879855,0.0012227357861912594,0.4100031277819213,0.008997757290374837,,,,0.25772944093759054,0.008835947746991793,0.6198384674758057,0.017198709543899918,0.016385743507442433,0.002716078358098081,0.048296473362351745,0.0015538717218489837,0.40254559873749207,0.010882343687310897,0.115,0.21463786172170962,0.4494029850746269,,,,,,,,,,0.1420746514551761,0.07272450801528493,0.1420746514551761,Baseline
+66,0,Grad. Ascent,Phi,0.01,5,4,1e-05,0.9473684210526316,0.006999999999999999,0.005366374921488416,0.33162692114515296,0.034602705500507855,0.3086568230497394,0.02469392939030263,0.00014492753623188405,0.00010196035806977516,0.3273254466202702,0.008285297737596824,0.045156695156695154,0.013729649816810394,0.38764041548587147,0.02982279652678528,0.3606459158735745,0.027334710961078278,0.0008036077487346345,0.0002297548355519693,0.34408214632182105,0.008165113849349167,0.17668430862926077,0.005137403940778033,0.3294075029061917,0.017063007349037172,0.0009094803730757254,0.00025547778244081303,0.028704598972948123,0.0007996272178659856,0.32993047398003306,0.005275591767363487,,,,0.16250982424789703,0.00499411861704525,0.5769310997983197,0.017711529027575522,0.00044359152207463083,0.0002348360930215928,0.03221285623428339,0.0011033966488544287,0.2996506352525866,0.0051596639245466355,0.125,0.19294319281521355,0.4469154228855721,,,,,,,,,,0.0878382826713132,0.007004792878636915,0.0878382826713132,Baseline
+42,0,Grad. Diff.,Phi,0.01,5,0,1e-05,0.9151450053705693,0.3973333333333333,0.04870415050172085,0.4678435778720407,0.032137010774723654,0.38071170295758916,0.02459418835823046,0.006365911256096586,0.0008343610756620373,0.3397004696498032,0.002636847356886148,0.7642450142450141,0.03790882473231565,0.5007235586805928,0.029071911794725465,0.4226639082672956,0.02477886455076359,0.008595846256573423,0.0006257596806937399,0.3311016865466536,0.004794229183144251,0.8721092880212769,0.012298979869899092,0.4744406979742168,0.014880458639131443,0.9025795877529031,0.003916338520805818,0.1442091384328562,0.003648217718380057,0.4256276909117877,0.006257025952016907,,,,0.8407838747624112,0.016036895405682237,0.48067977707224346,0.01625556138025168,0.8840307826757645,0.006614568352365256,0.16391258248782223,0.005011926277748944,0.4468026409556067,0.007465492502146892,0.39,4.894475734218317e-12,0.21830845771144278,,,,,,,,,,5.86730162802883e-14,0.5197875247816769,5.86730162802883e-14,Baseline
+43,0,Grad. Diff.,Phi,0.01,5,1,1e-05,0.9151450053705693,0.3473333333333333,0.04736177833589072,0.4658565743478556,0.032190529189740136,0.37902972462361545,0.024914625287162914,0.005572909503883965,0.0008129460970439699,0.34209716269796464,0.0035569512250199073,0.7474358974358973,0.03827063998189997,0.49612806638139373,0.029576486797460378,0.4226405600755504,0.025008676244423402,0.008032865943899542,0.0005621164819683218,0.32865105877525935,0.0038978426238417806,0.8258097903202785,0.013911671417644524,0.47413569492142965,0.01496968822128586,0.8868977975392949,0.004687838770332232,0.1356322976688059,0.003579559088357858,0.4315184634867408,0.006307123916588194,,,,0.6413835780521334,0.01682017466894216,0.48577763100969434,0.016418064941580508,0.7061440694141925,0.014532229391702015,0.12174831962148079,0.004028227849277732,0.44208240006432176,0.006829448495821801,0.38,1.1776592896645094e-11,0.22343283582089551,,,,,,,,,,2.9623487356936016e-13,0.5047352737509242,2.9623487356936016e-13,Baseline
+44,0,Grad. Diff.,Phi,0.01,5,2,1e-05,0.9151450053705693,0.24483333333333335,0.04215085342486489,0.4625359994576103,0.032621888404754114,0.3758971519871067,0.025677245003106298,0.003775727623708952,0.0006640346510903368,0.3500616432827556,0.006118399018951057,0.6762108262108263,0.041283143178581436,0.4889036615332099,0.02943054153484348,0.4236861963949195,0.026138168444356055,0.007372619834467888,0.0005829880943059497,0.33673263065716685,0.005253124826603497,0.6049396776020457,0.013976971298303968,0.4612535221474117,0.01530767798039172,0.7362495710152509,0.00823504442092462,0.09543865882076759,0.00257352751624662,0.4279937556012972,0.0058471810291254285,,,,0.467504242312318,0.013490950332617421,0.5008212466462312,0.016403700482951537,0.36403334928207415,0.016235857834725916,0.08644949076627832,0.0024920869852342598,0.4551870992489783,0.007053281002471597,0.345,9.774312579286385e-10,0.2507462686567164,,,,,,,,,,5.951062636424447e-11,0.4506655509391546,5.951062636424447e-11,Baseline
+45,0,Grad. Diff.,Phi,0.01,5,3,1e-05,0.9151450053705693,0.25066666666666665,0.04155906645353501,0.4522186890185522,0.03379806716500173,0.37163065732523054,0.026163742163857955,0.0038356477638539577,0.0006603467510802686,0.3799103276265034,0.007895020897578117,0.6021367521367521,0.04276479954276994,0.4827709140534684,0.029394363324439343,0.4223339028901742,0.026994260464550626,0.006631917105760502,0.0005881291691288757,0.35214358840198745,0.006904142661142702,0.5114967609955267,0.01188225990782738,0.44601523164022544,0.01569150751068399,0.5945503053901763,0.01000923752105491,0.07978642994593477,0.0018349873686373515,0.452711571950052,0.006338428504598451,,,,0.40989014418209946,0.011677618326873888,0.5299893964426794,0.016305335991572075,0.18144936255193322,0.012414113602790098,0.07658335918342869,0.00199129826860702,0.4881270035786109,0.007441988124251571,0.275,7.274398716819551e-07,0.2980099502487562,,,,,,,,,,4.6128382074998843e-07,0.43123040151855757,4.6128382074998843e-07,Baseline
+46,0,Grad. Diff.,Phi,0.01,5,4,1e-05,0.9151450053705693,0.212,0.039413546014379484,0.45064918770182705,0.034223452847729756,0.3693853118330659,0.026361926453242125,0.0032865313254840985,0.0006309876870707848,0.3993303075586398,0.009038594044291366,0.6106837606837607,0.04258106655203318,0.4804159197768448,0.029743575267259563,0.4217896553616893,0.027243025716062495,0.006715957730675074,0.0005872648636038186,0.3543789096754754,0.006734690172508233,0.5072339469034717,0.011995210499331867,0.44490751045661825,0.015743106866691527,0.570797476051256,0.010328660412022682,0.08016542054049272,0.002002839372378644,0.45708176744168477,0.006980054634977941,,,,0.395527053299195,0.011627848646539426,0.5437199008557633,0.016381511529563884,0.12948525108431583,0.010609580421746081,0.07608437028829533,0.0024358043498195715,0.49979007603264874,0.008432291993143413,0.245,5.180660338161201e-06,0.31417910447761194,,,,,,,,,,1.1150402145378719e-05,0.41466703707754604,1.1150402145378719e-05,Baseline
+72,0,KL Min.,Phi,0.01,5,0,1e-05,0.8678839957035446,0.4073333333333333,0.04890628707870714,0.46895545751340906,0.03207997819700763,0.3807860232150567,0.024589508967707528,0.006506889590530774,0.0008351364921554757,0.3364273150188554,0.00256961225665061,0.7727920227920227,0.03738327846205438,0.49975883125095255,0.029023278467080225,0.42347443380966066,0.02480494831953641,0.008658448792302847,0.0006230229367984266,0.3249002409958939,0.004029200497814564,0.8625211609105699,0.01252761288739136,0.472905162798203,0.014921074256770726,0.8991544656111204,0.0040813831635363625,0.14191658934091503,0.003527427539152901,0.4226968038809161,0.006248706436982362,,,,0.8415051252115527,0.015822463829402358,0.48103219042642215,0.016284674046883864,0.8739887868170981,0.007177742585447102,0.16364532509448323,0.004963007282731629,0.4380563297491344,0.006778505447397485,0.39,5.288020907026845e-12,0.21875621890547264,,,,,,,,,,5.86730162802883e-14,0.5215557241932791,5.86730162802883e-14,Baseline
+73,0,KL Min.,Phi,0.01,5,1,1e-05,0.8678839957035446,0.33233333333333337,0.04657628332477988,0.46406758093050593,0.03208968861791881,0.378204987775231,0.02501102177000738,0.005434478098201527,0.0008152768042422294,0.332541625242257,0.002382512986644175,0.7670940170940171,0.03702451857933602,0.4981194202876374,0.029189789077508254,0.4270124226268368,0.025532853965627128,0.008438522246750818,0.00060278426413987,0.32322164246050733,0.004028717682881305,0.7064093892130515,0.015082685497990907,0.46923033724039254,0.01495915728848286,0.8243124665383831,0.006745036307766154,0.11366100559311693,0.0032262541023935893,0.4024112557641747,0.0054078315311679075,,,,0.550586528259906,0.01587925449069617,0.4884893714926913,0.016177056402482636,0.5929438127792263,0.016478183846341768,0.10276832959759241,0.0033053068411054436,0.41192872182192913,0.0052643041532413224,0.39,5.1776563533332604e-11,0.23228855721393035,,,,,,,,,,5.86730162802883e-14,0.4939390933429211,5.86730162802883e-14,Baseline
+74,0,KL Min.,Phi,0.01,5,2,1e-05,0.8678839957035446,0.15783333333333333,0.03513678178694564,0.44880469697896536,0.03330190786006885,0.3695053509344211,0.026793042852428756,0.0024883713637950723,0.0005582520394266483,0.37583756281221703,0.009632572789225403,0.5636752136752137,0.04342640003071448,0.47170232334802753,0.029047550892587926,0.42134336876327244,0.02785996987560283,0.006075492986915766,0.0005733436262154558,0.32341987776091663,0.004862266282380578,0.41647037679034904,0.009409339329114003,0.42045584458400276,0.015486731485031775,0.3426740213967376,0.010722104005944182,0.06624829974475784,0.0024000741537383245,0.40407332380582056,0.005672566558969576,,,,0.3954147946279467,0.011237187721797824,0.5262106743299542,0.01605648314875436,0.16433360275066503,0.011908963011260066,0.07390941815371004,0.0021937924117020396,0.4334637178689931,0.0069395438149276786,0.27,5.339896563528368e-07,0.2955721393034826,,,,,,,,,,8.056669472003223e-07,0.3566418642799969,8.056669472003223e-07,Baseline
+75,0,KL Min.,Phi,0.01,5,3,1e-05,0.8678839957035446,0.015333333333333332,0.007962031109832426,0.4028683587264754,0.03200797177096022,0.34005911321481364,0.026561192619630712,0.0002825899164773845,0.00013923913228522386,0.506884036880587,0.013238950409730204,0.22749287749287747,0.03503451733772163,0.4242603972960256,0.03069538570169358,0.39826567687915027,0.029062295812726124,0.002655673088603394,0.00041719267347636125,0.4259448142323811,0.014104314804628487,0.30317311642168526,0.008225994000437796,0.35188362225687625,0.01614564168189605,0.055630761897987234,0.005027591203888491,0.053512607826809076,0.0034524105553360796,0.47580466523429077,0.008836022468011344,,,,0.28760103073256016,0.009621335604118078,0.6120420372409806,0.017022145032074425,0.023781530102960104,0.0035930177435653512,0.05782994815787557,0.003501785056211076,0.4599772267791548,0.010699873314550502,0.115,0.08425016422364179,0.42960199004975125,,,,,,,,,,0.1420746514551761,0.08652191515378063,0.1420746514551761,Baseline
+76,0,KL Min.,Phi,0.01,5,4,1e-05,0.8678839957035446,0.002,0.001999999999999999,0.3553673668831616,0.03292035490154195,0.3165075376792712,0.02470159451223925,7.142857142857142e-05,7.142857142857143e-05,0.4175725218761348,0.012571455541940033,0.0779202279202279,0.019093212206409506,0.39577333969288264,0.031148681295742026,0.3814122528427597,0.027440055776921112,0.0011645059167925693,0.0002617646011644419,0.40376467560787016,0.011295870546941068,0.21474838348528966,0.0059902324388530085,0.3249776768208696,0.01679551500230718,0.004729788271612479,0.0007870849443046907,0.035124776240938044,0.0010017132999040517,0.41598280561224865,0.007480651046526885,,,,0.19625366054827842,0.006357602798082099,0.6019272754736927,0.01752453550368034,0.0020996731725884065,0.0007612685661735476,0.03873003132002448,0.0012853609631510894,0.3980496995718481,0.009124953134032337,0.1,0.2737389674190561,0.4553731343283582,,,,,,,,,,0.2704743832803917,0.012111175015277706,0.2704743832803917,Baseline
+52,0,Pref. Opt.,Phi,0.01,5,0,1e-05,0.9151450053705693,0.38733333333333336,0.04848034069953959,0.47077325716159346,0.0322983226826558,0.38233544228046357,0.024487486116936023,0.006224131060770215,0.0008329441836443652,0.3311313411613099,0.002438340165616331,0.7642450142450141,0.03790882473231565,0.504397096959188,0.02871043114986494,0.4232827365297125,0.024480867210747437,0.008590496228438141,0.0006238220996465724,0.3192531875703322,0.00396495767718937,0.8650445849866851,0.013004283881015261,0.46975287662066717,0.014755165836636464,0.9009263739228758,0.0038855718112490918,0.14233980391126533,0.0036458015279000576,0.41758241080386316,0.0061939075359923185,,,,0.8785970368046897,0.014571104369632755,0.4847094697929499,0.016147036195730703,0.8989153825341245,0.005571797458172596,0.17251406425110488,0.005130522098431152,0.4384495974595747,0.007467537247416522,0.38,8.606318711178264e-12,0.22159203980099504,,,,,,,,,,2.9623487356936016e-13,0.5181432182138059,2.9623487356936016e-13,Baseline
+53,0,Pref. Opt.,Phi,0.01,5,1,1e-05,0.9151450053705693,0.3106666666666667,0.045760514544288325,0.449021752363992,0.03166147493306669,0.37036629188340287,0.023058356017181193,0.005156394247336039,0.0008051428051619369,0.3499675601121971,0.005917068032809371,0.7094017094017094,0.041322819670542126,0.5082493937967322,0.02703999671345648,0.4173696054826313,0.023094151301457193,0.007904739096664714,0.0006246132203410139,0.3501697147138109,0.006231350800435453,0.5176678951269774,0.020466395195891728,0.44797862969742036,0.014196830878312808,0.80351049133941,0.005369812311164263,0.07981394704024042,0.0034348834320896667,0.4530654723845703,0.00833982479157137,,,,0.3456792256646202,0.02072274301227613,0.5160838994039727,0.015970211373317677,0.7622839465674903,0.007364296453277785,0.06841574910892625,0.00446810380014485,0.4771874157568081,0.0107467868537877,0.32,2.5711329376685905e-09,0.2571144278606965,,,,,,,,,,1.8694649189611297e-09,0.4656543567037805,1.8694649189611297e-09,Baseline
+54,0,Pref. Opt.,Phi,0.01,5,2,1e-05,0.9151450053705693,0.17733333333333334,0.03777445023966717,0.43105403567531053,0.03132970212117321,0.3617102253689021,0.022473645018072495,0.0029110218982668513,0.0006585486177014463,0.37678644033452957,0.014937713622423405,0.6766381766381766,0.04259769328697061,0.499037600014566,0.027164826839664882,0.4124155062951512,0.022265558553862155,0.007546741827556888,0.0006286733282296054,0.34394485963113075,0.010014142273038209,0.3376444741658668,0.01667198132129853,0.4394729954443767,0.014006406535919197,0.7536458413813074,0.0061096575486654535,0.054499814795241595,0.002889757589839446,0.5140921887597085,0.008998708664478402,,,,0.2185774077863272,0.0093122165661745,0.525439985056896,0.01563000353579026,0.6757358105983836,0.007732263757350835,0.043656418792839366,0.00236879274776873,0.549872205907207,0.010050968785150908,0.31,3.582459415247659e-08,0.2753233830845771,,,,,,,,,,6.865581733487208e-09,0.38802873821884926,6.865581733487208e-09,Baseline
+55,0,Pref. Opt.,Phi,0.01,5,3,1e-05,0.9151450053705693,0.2606666666666667,0.04333043243333383,0.4368441181991892,0.0314334754633251,0.36519754975692126,0.022584344830396244,0.00431047181235687,0.000754822151770228,0.33054180843487785,0.010963783665047258,0.7065527065527065,0.04164330677576536,0.5029415692144049,0.02723380079782264,0.4161844633832956,0.02214994286136674,0.007690487137519026,0.0006328094000606604,0.31136389075046145,0.00708691778999871,0.48066021426910643,0.020109211153698202,0.4451154232504332,0.013934350734310845,0.7978543138750059,0.00536444490855783,0.07594731510535416,0.0035597390081840725,0.48338294610137117,0.0095435986970563,,,,0.21215981442010923,0.009550072386650968,0.5221263361368914,0.015613002687054296,0.7087758466564626,0.007474080250318148,0.0429872559165959,0.0024591884780993676,0.4873829223001848,0.01219178289721992,0.32,1.7041933048529214e-08,0.2700497512437811,,,,,,,,,,1.8694649189611297e-09,0.4444322693477377,1.8694649189611297e-09,Baseline
+56,0,Pref. Opt.,Phi,0.01,5,4,1e-05,0.9151450053705693,0.2756666666666667,0.04384662330054288,0.4400784419453887,0.03151288719072576,0.3665461930904549,0.02267574042080427,0.004509700848657644,0.0007599026971096693,0.321622807204187,0.007642186916448955,0.7364672364672364,0.04001870838730468,0.5019798132280175,0.027355076580149035,0.41516243280871923,0.02206562214156096,0.008038726792771947,0.0006290897388259802,0.3166833482357607,0.0061427314831959574,0.5530977000457807,0.020689279854693463,0.44772954835910145,0.01398835914622035,0.8141930350584041,0.0050901634917780245,0.08705088747430882,0.0036998080583930766,0.4477251066365466,0.00887786542772537,,,,0.22430610260291634,0.01172762080648869,0.51995174384652,0.01567541348276156,0.7246818215455729,0.00737963992864854,0.04457842146545089,0.0027130341265047806,0.4448460324395553,0.012060145743815337,0.315,9.018692040843481e-09,0.2656218905472637,,,,,,,,,,3.602472544567432e-09,0.4578660887445282,3.602472544567432e-09,Baseline
+82,0,Retain Model,Phi,0.01,5,-1,2e-05,0.0,0.42483333333333334,0.04875756869502934,0.45316206859199293,0.03264141366244881,0.37652852177657875,0.019938203231217873,0.006830683283714753,0.0008441486901540186,0.38193041689621005,0.00508030085542245,0.7514245014245015,0.03841420788700116,0.4996699066372895,0.02988294175220104,0.415358088017408,0.01856589722762314,0.008419340748676295,0.000614840909663249,0.3716823986479536,0.005860168184119516,0.8730404742518771,0.011887122608792737,0.47646922056672225,0.014784468854096159,0.9029671577901152,0.0037056191445714475,0.14748570805725,0.003641886728964624,0.49837190765545397,0.007436411553182218,,,,0.4287828600171711,0.011889233433531439,0.6308750426217175,0.016057591005249512,0.13932983163434137,0.010164844693333376,0.0794925659968092,0.001858261239460288,0.5170377343029624,0.008508323382192614,0.0,1.0,0.0,,,,,,,,,,1.0,0.5199978641100643,1.0,Baseline