Initial commit (#1)

- Initial commit (d4c52034a11be912014a2ecc30b9c8a9d4ca6ba3)


Co-authored-by: Koshti <[email protected]>

app.py

@@ -0,0 +1,128 @@
+import gradio as gr
+
+from src.assets.text_content import TITLE, INTRODUCTION_TEXT
+from src.utils import get_data, compare_plots, filter_search
+
+############################ For Leaderboards #############################
+DATA_PATH = 'versions'
+latest_flag = True #Set flag to iclude latest data in Details and Versions Tab
+latest_df, latest_vname, previous_df, previous_vname = get_data(DATA_PATH, latest_flag)
+
+global prev_df
+prev_df = previous_df[0]
+def select_prev_df(name):
+    ind = previous_vname.index(name)
+    prev_df = previous_df[ind]
+    return prev_df
+
+############################ For Plots ####################################
+global plot_df, MODEL_COLS
+plot_df = latest_df[0]
+MODEL_COLS = list(plot_df['Model'].unique())
+
+
+############# MAIN APPLICATION ######################
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🥇 Clem Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                search_bar = gr.Textbox(
+                    placeholder=" 🔍 Search for models - separate multiple queries with `;` and press ENTER...",
+                    show_label=False,
+                    elem_id="search-bar",
+                )
+                        
+            leaderboard_table = gr.components.Dataframe(
+                value=latest_df[0],
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Add a dummy leaderboard to handle search queries from the latest_df and not update latest_df
+            dummy_leaderboard_table = gr.components.Dataframe(
+                value=latest_df[0],
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=False,
+            )
+                
+            search_bar.submit(
+                filter_search,
+                [dummy_leaderboard_table, search_bar],
+                leaderboard_table,
+                queue=True
+            )
+        with gr.TabItem("📈 Plot", id=3):
+            with gr.Row():
+                model_cols = gr.CheckboxGroup(
+                    MODEL_COLS, 
+                    label="Select Models 🤖", 
+                    value=[],
+                    elem_id="column-select",
+                    interactive=True,
+                )
+
+            with gr.Row():
+                plot_grdf = gr.DataFrame(
+                    value=plot_df,
+                    visible=False
+                )
+            with gr.Row():
+                # Output block for the plot
+                plot_output = gr.Plot()
+
+            model_cols.change(
+                compare_plots,
+                [plot_grdf, model_cols],
+                plot_output,
+                queue=True
+            )
+
+        with gr.TabItem("🔄 Versions and Details", elem_id="details", id=2):
+            with gr.Row():
+                ver_selection = gr.Dropdown(
+                    previous_vname, label="Select Version 🕹️", value=previous_vname[0]
+                )
+            with gr.Row():
+                search_bar_prev = gr.Textbox(
+                    placeholder=" 🔍 Search for models - separate multiple queries with `;` and press ENTER...",
+                    show_label=False,
+                    elem_id="search-bar-2",
+                )
+
+            prev_table = gr.components.Dataframe(
+                value=prev_df,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            dummy_prev_table = gr.components.Dataframe(
+                value=prev_df,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=False,
+            )
+
+            search_bar_prev.submit(
+                filter_search,
+                [dummy_prev_table, search_bar_prev],
+                prev_table,
+                queue=True
+            )
+
+            ver_selection.change(
+                select_prev_df,
+                [ver_selection],
+                prev_table,
+                queue=True
+            )
+
+    demo.load()
+demo.queue()
+demo.launch()

requirements.txt

@@ -0,0 +1,75 @@
+accelerate==0.23.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.5.0
+gradio==3.43.2
+gradio-client==0.5.0
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.16.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers@git+https://github.com/clefourrier/transformers.git
+tokenizers==0.14
+#tokenizers==0.14.1 wait for tokenizers patch in dependencies with hf_hub
+#transformers==4.34
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2
+hf_transfer==0.1.3

src/assets/text_content.py

@@ -0,0 +1,18 @@
+TITLE = """<h1 align="center" id="space-title"> 🏆 CLEM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = """
+🔝 The CLEM Leaderboard aims to track, rank and evaluate current cLLMs (chat-optimized Large Language Models, “clems”) as described in [Clembench: Using Game Play to Evaluate Chat-Optimized Language Models as Conversational Agents](https://arxiv.org/abs/2305.13455).
+"""
+
+SHORT_NAMES = {
+    "t0.0": "",
+    "claude-v1.3-": "cl",
+    "gpt-3.5-turbo-": "3.5",
+    "gpt-4-": "4",
+    "text-davinci-003-": "3",
+    "luminous-supreme-": "lm",
+    "koala-13b-": "ko",
+    "falcon-40b-": "flc",
+    "oasst-12b-": "ost",
+    "vicuna-13b-": "vcn"
+}

src/utils.py

@@ -0,0 +1,217 @@
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+from src.assets.text_content import SHORT_NAMES
+
+def update_cols(df: pd.DataFrame) -> pd.DataFrame:
+    '''
+    Change three header rows to a single header row
+    Args:
+        df: Raw dataframe containing 3 separate header rows
+            Remove this function if the dataframe has only one header row
+
+    Returns:
+        df: Updated dataframe which has only 1 header row instead of 3
+    '''
+    default_cols = list(df.columns)
+
+    # First 4 columns are initalised in 'update', Append additional columns for games Model, Clemscore, ALL(PLayed) and ALL(Main Score)
+    update = ['Model', 'Clemscore', 'All(Played)', 'All(Quality Score)']
+    game_metrics = default_cols[4:]
+
+    # Change columns Names for each Game
+    for i in range(len(game_metrics)):
+        if i%3 == 0:
+            game = game_metrics[i]
+            update.append(str(game).capitalize() + "(Played)")
+            update.append(str(game).capitalize() + "(Quality Score)") 
+            update.append(str(game).capitalize() + "(Quality Score[std])")
+
+    # Create a dict to change names of the columns
+    map_cols = {}
+    for i in range(len(default_cols)):
+        map_cols[default_cols[i]] = str(update[i])
+
+    df = df.rename(columns=map_cols)
+    df = df.iloc[2:]
+
+    return df
+
+def process_df(df: pd.DataFrame) -> pd.DataFrame:
+    '''
+    Process dataframe - Remove repition in model names, convert datatypes to sort by "float" instead of "str"
+    Args:
+        df: Unprocessed Dataframe (after using update_cols)
+    Returns:
+        df: Processed Dataframe
+    '''
+
+    # Change column type to float from str
+    list_column_names = list(df.columns)
+    model_col_name = list_column_names[0]
+    for col in list_column_names:
+        if col != model_col_name:
+            df[col] = df[col].astype(float)
+
+    # Remove repetition in model names, if any
+    models_list = []
+    for i in range(len(df)):
+        model_name = df.iloc[i][model_col_name]
+        splits = model_name.split('--')
+        splits = [split.replace('-t0.0', '') for split in splits] # Comment to not remove -t0.0
+        if splits[0] == splits[1]:
+            models_list.append(splits[0])
+        else:
+            models_list.append(splits[0] + "--" + splits[1])
+    df[model_col_name] = models_list
+    
+    return df
+
+def get_data(path: str, flag: bool):
+    '''
+    Get a list of all version names and respective Dataframes 
+    Args: 
+        path: Path to the directory containing CSVs of different versions -> v0.9.csv, v1.0.csv, ....
+        flag: Set this flag to include the latest version in Details and Versions tab
+    Returns: 
+        latest_df: singular list containing dataframe of the latest version of the leaderboard with only 4 columns 
+        latest_vname: list of the name of latest version 
+        previous_df: list of dataframes for previous versions (can skip latest version if required) 
+        previous_vname: list of the names for the previous versions (INCLUDED IN Details and Versions Tab)
+
+    '''
+    # Check if Directory is empty
+    list_versions = os.listdir(path)
+    if not list_versions:
+        print("Directory is empty")
+
+    else:
+        files = [file for file in list_versions if file.endswith('.csv')]
+        files.sort(reverse=True)
+        file_names = [os.path.splitext(file)[0] for file in files]
+
+        DFS = []
+        for file in files:
+            df = pd.read_csv(os.path.join(path, file))
+            df = update_cols(df) # Remove if by default there is only one header row
+            df = process_df(df) # Process Dataframe
+            df = df.sort_values(by=list(df.columns)[1], ascending=False) # Sort by clemscore
+            DFS.append(df)
+
+        # Only keep relavant columns for the main leaderboard
+        latest_df_dummy = DFS[0]
+        all_columns = list(latest_df_dummy.columns)
+        keep_columns = all_columns[0:4]
+        latest_df_dummy = latest_df_dummy.drop(columns=[c for c in all_columns if c not in keep_columns])
+
+        latest_df = [latest_df_dummy]
+        latest_vname = [file_names[0]]
+        previous_df = []
+        previous_vname = []
+        for df, name in zip(DFS, file_names):
+            previous_df.append(df)
+            previous_vname.append(name) 
+        
+        if not flag:
+            previous_df.pop(0)
+            previous_vname.pop(0)
+
+        return latest_df, latest_vname, previous_df, previous_vname
+    
+    return None
+
+
+# ['Model', 'Clemscore', 'All(Played)', 'All(Quality Score)']
+def compare_plots(df: pd.DataFrame, LIST: list):
+    '''
+    Quality Score v/s % Played plot by selecting models
+    Args:
+        LIST: The list of models to show in the plot, updated from frontend
+    Returns:
+        fig: The plot
+    '''
+    short_names = label_map(LIST)
+
+    list_columns = list(df.columns)
+    df = df[df[list_columns[0]].isin(LIST)]
+
+    X = df[list_columns[2]]
+    fig, ax = plt.subplots()
+    for model in LIST:
+        short = short_names[model][0]
+        same_flag = short_names[model][1]
+        model_df = df[df[list_columns[0]] == model]
+        x = model_df[list_columns[2]]
+        y = model_df[list_columns[3]]
+        color = plt.cm.rainbow(x / max(X))  # Use a colormap for different colors
+        plt.scatter(x, y, color=color)
+        if same_flag:
+            plt.annotate(f'{short}', (x, y), textcoords="offset points", xytext=(0, -15), ha='center', rotation=0)
+        else:
+            plt.annotate(f'{short}', (x, y), textcoords="offset points", xytext=(20, -3), ha='center', rotation=0)
+    ax.grid(which='both', color='grey', linewidth=1, linestyle='-', alpha=0.2)
+    ax.set_xticks(np.arange(0,110,10))
+    plt.xlim(-10, 110)
+    plt.ylim(-10, 110)
+    plt.xlabel('% Played')
+    plt.ylabel('Quality Score')
+    plt.title('Overview of benchmark results')
+    plt.show()
+
+    return fig
+
+
+def label_map(model_list: list) -> dict:
+    '''
+    Generate a map from long names to short names, to plot them in frontend graph
+    Define the short names in src/assets/text_content.py
+    Args: 
+        model_list: A list of long model names
+    Returns:
+        short_name: A map from long to list of short name + indication if models are same or different
+    '''
+    short_name = {}
+    for model_name in model_list:
+        splits = model_name.split('--')
+        if len(splits) != 1:
+            splits[0] = SHORT_NAMES[splits[0] + '-']
+            splits[1] = SHORT_NAMES[splits[1] + '-']
+            # Define the short name and indicate there are two different models
+            short_name[model_name] = [splits[0] + '--' + splits[1], 0]
+        else:
+            splits[0] = SHORT_NAMES[splits[0] + '-']
+            # Define the short name and indicate both models are same 
+            short_name[model_name] = [splits[0], 1]
+
+    return short_name
+
+def filter_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    '''
+    Filter the dataframe based on the search query
+    Args:
+        df: Unfiltered dataframe
+        query: a string of queries separated by ";"
+    Return:
+        filtered_df: Dataframe containing searched queries in the 'Model' column 
+    '''
+    queries = query.split(';')
+    list_cols = list(df.columns)
+    df_len = len(df)
+    filtered_models = []
+    models_list = list(df[list_cols[0]])
+    for q in queries:
+        q = q.lower()
+        for i in range(df_len):
+            model_name = models_list[i]
+            if q in model_name.lower():
+                filtered_models.append(model_name) # Append model names containing query q
+
+    filtered_df = df[df[list_cols[0]].isin(filtered_models)]
+
+    if query == "":
+        return df
+
+    return filtered_df
+    

versions/v0.7.csv

@@ -0,0 +1,14 @@
+,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
+,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
+model,,,,,,,,,,,,,,,,,,,,,,,,
+claude-v1.3-t0.0--claude-v1.3-t0.0,37.07,74.76,49.58,0.0,,,100.0,84.87,18.87,100.0,82.5,38.48,76.92,68.75,38.71,100.0,0.0,0.0,100.0,30.56,40.13,46.43,30.77,48.04
+falcon-40b-t0.0--falcon-40b-t0.0,0.71,0.95,75.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,3.33,50.0,,3.33,100.0,
+gpt-3.5-turbo-t0.0--gpt-3.5-turbo-t0.0,37.02,85.86,43.12,97.5,60.28,25.95,64.0,72.83,13.07,100.0,55.0,50.38,69.49,71.95,44.79,100.0,0.0,0.0,93.33,28.57,46.0,76.67,13.19,30.16
+gpt-3.5-turbo-t0.0--gpt-4-t0.0,42.39,86.75,48.87,97.5,64.95,25.45,,,,100.0,57.5,50.06,69.49,62.6,45.15,,,,,,,80.0,10.42,17.42
+gpt-4-t0.0--gpt-3.5-turbo-t0.0,55.62,82.78,67.19,65.0,81.0,21.54,,,,100.0,47.5,50.57,66.1,93.59,23.45,,,,,,,100.0,46.67,42.92
+gpt-4-t0.0--gpt-4-t0.0,7.77,96.06,61.93,77.5,89.06,22.28,100.0,90.79,8.2,100.0,75.0,43.85,94.92,76.19,37.45,100.0,3.67,8.4,100.0,49.67,42.09,100.0,49.11,38.46
+koala-13b-t0.0--koala-13b-t0.0,1.48,14.76,10.0,0.0,,,0.0,,,0.0,,,0.0,,,86.67,0.0,0.0,16.67,20.0,44.72,0.0,,
+luminous-supreme-t0.0--luminous-supreme-t0.0,0.0,16.24,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,10.34,0.0,0.0
+oasst-12b-t0.0--oasst-12b-t0.0,1.74,20.85,8.33,0.0,,,0.0,,,15.0,33.33,51.64,0.0,,,100.0,0.0,0.0,16.67,0.0,0.0,14.29,0.0,0.0
+text-davinci-003-t0.0--text-davinci-003-t0.0,15.78,44.5,35.46,57.5,38.7,27.78,16.0,14.1,25.21,82.5,36.36,48.85,28.81,76.47,43.72,66.67,1.25,5.59,36.67,31.36,38.99,23.33,50.0,50.0
+vicuna-13b-t0.0--vicuna-13b-t0.0,4.24,13.58,31.25,0.0,,,0.0,,,0.0,,,5.08,100.0,0.0,56.67,0.0,0.0,13.33,25.0,50.0,20.0,0.0,0.0

versions/v0.8.csv

@@ -0,0 +1,14 @@
+,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
+,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
+model,,,,,,,,,,,,,,,,,,,,,,,,
+claude-v1.3-t0.0--claude-v1.3-t0.0,37.07,74.76,49.58,0.0,,,100.0,84.87,18.87,100.0,82.5,38.48,76.92,68.75,38.71,100.0,0.0,0.0,100.0,30.56,40.13,46.43,30.77,48.04
+falcon-40b-t0.0--falcon-40b-t0.0,0.71,0.95,75.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,3.33,50.0,,3.33,100.0,
+gpt-3.5-turbo-t0.0--gpt-3.5-turbo-t0.0,37.02,85.86,43.12,97.5,60.28,25.95,64.0,72.83,13.07,100.0,55.0,50.38,69.49,71.95,44.79,100.0,0.0,0.0,93.33,28.57,46.0,76.67,13.19,30.16
+gpt-3.5-turbo-t0.0--gpt-4-t0.0,42.39,86.75,48.87,97.5,64.95,25.45,,,,100.0,57.5,50.06,69.49,62.6,45.15,,,,,,,80.0,10.42,17.42
+gpt-4-t0.0--gpt-3.5-turbo-t0.0,55.62,82.78,67.19,65.0,81.0,21.54,,,,100.0,47.5,50.57,66.1,93.59,23.45,,,,,,,100.0,46.67,42.92
+gpt-4-t0.0--gpt-4-t0.0,8.88,96.06,61.93,77.5,89.06,22.28,100.0,90.79,8.2,100.0,75.0,43.85,94.92,76.19,37.45,100.0,3.67,8.4,100.0,49.67,42.09,100.0,49.11,38.46
+koala-13b-t0.0--koala-13b-t0.0,1.48,14.76,10.0,0.0,,,0.0,,,0.0,,,0.0,,,86.67,0.0,0.0,16.67,20.0,44.72,0.0,,
+luminous-supreme-t0.0--luminous-supreme-t0.0,0.0,16.24,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,10.34,0.0,0.0
+oasst-12b-t0.0--oasst-12b-t0.0,1.74,20.85,8.33,0.0,,,0.0,,,15.0,33.33,51.64,0.0,,,100.0,0.0,0.0,16.67,0.0,0.0,14.29,0.0,0.0
+text-davinci-003-t0.0--text-davinci-003-t0.0,15.78,44.5,35.46,57.5,38.7,27.78,16.0,14.1,25.21,82.5,36.36,48.85,28.81,76.47,43.72,66.67,1.25,5.59,36.67,31.36,38.99,23.33,50.0,50.0
+vicuna-13b-t0.0--vicuna-13b-t0.0,4.24,13.58,31.25,0.0,,,0.0,,,0.0,,,5.08,100.0,0.0,56.67,0.0,0.0,13.33,25.0,50.0,20.0,0.0,0.0

versions/v0.9.csv

@@ -0,0 +1,14 @@
+,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
+,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
+model,,,,,,,,,,,,,,,,,,,,,,,,
+claude-v1.3-t0.0--claude-v1.3-t0.0,37.07,74.76,49.58,0.0,,,100.0,84.87,18.87,100.0,82.5,38.48,76.92,68.75,38.71,100.0,0.0,0.0,100.0,30.56,40.13,46.43,30.77,48.04
+falcon-40b-t0.0--falcon-40b-t0.0,0.71,0.95,75.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,3.33,50.0,,3.33,100.0,
+gpt-3.5-turbo-t0.0--gpt-3.5-turbo-t0.0,37.02,85.86,43.12,97.5,60.28,25.95,64.0,72.83,13.07,100.0,55.0,50.38,69.49,71.95,44.79,100.0,0.0,0.0,93.33,28.57,46.0,76.67,13.19,30.16
+gpt-3.5-turbo-t0.0--gpt-4-t0.0,42.39,86.75,48.87,97.5,64.95,25.45,,,,100.0,57.5,50.06,69.49,62.6,45.15,,,,,,,80.0,10.42,17.42
+gpt-4-t0.0--gpt-3.5-turbo-t0.0,55.62,82.78,67.19,65.0,81.0,21.54,,,,100.0,47.5,50.57,66.1,93.59,23.45,,,,,,,100.0,46.67,42.92
+gpt-4-t0.0--gpt-4-t0.0,9.99,96.06,61.93,77.5,89.06,22.28,100.0,90.79,8.2,100.0,75.0,43.85,94.92,76.19,37.45,100.0,3.67,8.4,100.0,49.67,42.09,100.0,49.11,38.46
+koala-13b-t0.0--koala-13b-t0.0,1.48,14.76,10.0,0.0,,,0.0,,,0.0,,,0.0,,,86.67,0.0,0.0,16.67,20.0,44.72,0.0,,
+luminous-supreme-t0.0--luminous-supreme-t0.0,0.0,16.24,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,10.34,0.0,0.0
+oasst-12b-t0.0--oasst-12b-t0.0,1.74,20.85,8.33,0.0,,,0.0,,,15.0,33.33,51.64,0.0,,,100.0,0.0,0.0,16.67,0.0,0.0,14.29,0.0,0.0
+text-davinci-003-t0.0--text-davinci-003-t0.0,15.78,44.5,35.46,57.5,38.7,27.78,16.0,14.1,25.21,82.5,36.36,48.85,28.81,76.47,43.72,66.67,1.25,5.59,36.67,31.36,38.99,23.33,50.0,50.0
+vicuna-13b-t0.0--vicuna-13b-t0.0,4.24,13.58,31.25,0.0,,,0.0,,,0.0,,,5.08,100.0,0.0,56.67,0.0,0.0,13.33,25.0,50.0,20.0,0.0,0.0

versions/v1.0.csv

@@ -0,0 +1,14 @@
+,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
+,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
+model,,,,,,,,,,,,,,,,,,,,,,,,
+claude-v1.3-t0.0--claude-v1.3-t0.0,37.07,74.76,49.58,0.0,,,100.0,84.87,18.87,100.0,82.5,38.48,76.92,68.75,38.71,100.0,0.0,0.0,100.0,30.56,40.13,46.43,30.77,48.04
+falcon-40b-t0.0--falcon-40b-t0.0,0.71,0.95,75.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,3.33,50.0,,3.33,100.0,
+gpt-3.5-turbo-t0.0--gpt-3.5-turbo-t0.0,37.02,85.86,43.12,97.5,60.28,25.95,64.0,72.83,13.07,100.0,55.0,50.38,69.49,71.95,44.79,100.0,0.0,0.0,93.33,28.57,46.0,76.67,13.19,30.16
+gpt-3.5-turbo-t0.0--gpt-4-t0.0,42.39,86.75,48.87,97.5,64.95,25.45,,,,100.0,57.5,50.06,69.49,62.6,45.15,,,,,,,80.0,10.42,17.42
+gpt-4-t0.0--gpt-3.5-turbo-t0.0,55.62,82.78,67.19,65.0,81.0,21.54,,,,100.0,47.5,50.57,66.1,93.59,23.45,,,,,,,100.0,46.67,42.92
+gpt-4-t0.0--gpt-4-t0.0,59.49,96.06,61.93,77.5,89.06,22.28,100.0,90.79,8.2,100.0,75.0,43.85,94.92,76.19,37.45,100.0,3.67,8.4,100.0,49.67,42.09,100.0,49.11,38.46
+koala-13b-t0.0--koala-13b-t0.0,1.48,14.76,10.0,0.0,,,0.0,,,0.0,,,0.0,,,86.67,0.0,0.0,16.67,20.0,44.72,0.0,,
+luminous-supreme-t0.0--luminous-supreme-t0.0,0.0,16.24,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,10.34,0.0,0.0
+oasst-12b-t0.0--oasst-12b-t0.0,1.74,20.85,8.33,0.0,,,0.0,,,15.0,33.33,51.64,0.0,,,100.0,0.0,0.0,16.67,0.0,0.0,14.29,0.0,0.0
+text-davinci-003-t0.0--text-davinci-003-t0.0,15.78,44.5,35.46,57.5,38.7,27.78,16.0,14.1,25.21,82.5,36.36,48.85,28.81,76.47,43.72,66.67,1.25,5.59,36.67,31.36,38.99,23.33,50.0,50.0
+vicuna-13b-t0.0--vicuna-13b-t0.0,4.24,13.58,31.25,0.0,,,0.0,,,0.0,,,5.08,100.0,0.0,56.67,0.0,0.0,13.33,25.0,50.0,20.0,0.0,0.0