Spaces:

Koshti10
/

TestLLMCalc

Sleeping

File size: 11,473 Bytes

ef818ff

import pandas as pd
import gradio as gr
import os
from gradio_rangeslider import RangeSlider

from src.filter_utils import filter, filter_cols

# Main Leaderboard containing everything
text_leaderboard = pd.read_csv(os.path.join('assets', 'merged_data.csv'))
text_leaderboard = text_leaderboard.sort_values(by='Clemscore', ascending=False)  

open_weight_df = text_leaderboard[text_leaderboard['Open Weight'] == True]
if not open_weight_df.empty:  # Check if filtered df is non-empty
    max_parameter_size = open_weight_df['Parameters (B)'].max()

# Short leaderboard containing fixed columns
short_leaderboard = filter_cols(text_leaderboard)

## Extract data
langs = []
licenses = []
ip_prices = []
op_prices = []
latencies = []
parameters = []
contexts = []
dates = []

for i in range(len(text_leaderboard)):
    lang_splits = text_leaderboard.iloc[i]['Languages'].split(',')
    lang_splits = [s.strip() for s in lang_splits]
    langs += lang_splits
    license_name = text_leaderboard.iloc[i]['License Name']

    licenses.append(license_name)
    ip_prices.append(text_leaderboard.iloc[i]['Input $/1M tokens'])
    op_prices.append(text_leaderboard.iloc[i]['Output $/1M tokens'])
    latencies.append(text_leaderboard.iloc[i]['Latency (s)'])
    parameters.append(text_leaderboard.iloc[i]['Parameters (B)'])
    contexts.append(text_leaderboard.iloc[i]['Context Size (k)'])
    dates.append(text_leaderboard.iloc[i]['Release Date'])


langs = list(set(langs))
langs.sort()

licenses = list(set(licenses))
licenses.sort()

max_input_price = max(ip_prices)
max_output_price = max(op_prices)
max_latency = max(latencies)

min_parameters = 0 if pd.isna(min(parameters)) else min(parameters)
max_parameter = max_parameter_size
parameter_step = 1
print(f"MIN {min_parameters}, MAX {max_parameter}")

min_context = min(contexts)
max_context = max(contexts)
context_step = 8

min_date = min(dates)
max_date = max(dates)

TITLE = """<h1 align="center" id="space-title"> LLM Calculator ⚖️⚡ 📏💰</h1>"""
CSS = """
#double-slider-1 {height: 100px}
#double-slider-2 {height: 100px}
#double-slider-3 {height: 100px}
#double-slider-4 {height: 100px}
"""

llm_calc_app = gr.Blocks(css=CSS)
with llm_calc_app:

    gr.HTML(TITLE)

    ##################################################

    with gr.Row():

        #####################################
        # First Column
        ####################################
        ## Language Select
        with gr.Column():

            with gr.Row():
                lang_dropdown = gr.Dropdown(
                    choices=langs,
                    value=[],
                    multiselect=True,
                    label="Select Languages 🗣️"
                )

            with gr.Row():
                start_date = gr.DateTime(
                    value=min_date,
                    type="string",
                    label="Release Date Range 📅 - Start Date"
                )

                end_date = gr.DateTime(
                    value=max_date,
                    type="string",
                    label="End Date"
                )

            # Multiodality Select
            with gr.Row():
                multimodal_checkbox = gr.CheckboxGroup(
                    choices=['Image', 'Multi-Image', 'Audio', 'Video'],
                    value=[],
                    label="Select Additional Modalities 📷🎧🎬",
                )

            # Open/Commercial Selection
            with gr.Row():
                open_weight_checkbox = gr.CheckboxGroup(
                    choices=['Open', 'Commercial'],
                    value=['Open', 'Commercial'],
                    label="Filter by Model Type 🔓 💼",
                )    

            # License selection
            with gr.Row():
                license_checkbox = gr.CheckboxGroup(
                    choices=licenses,
                    value=licenses,
                    label="License Type 🛡️",
                )    
        
        #############################################################
        # Second Column
        #############################################################
        with gr.Column():

            ####### LOG SLIDER 1 ###########
            with gr.Row():
                parameter_slider = RangeSlider(
                    minimum=0, 
                    maximum=max_parameter, 
                    label=f"Select Parameter Range 🔍 {int(min_parameters)}B - {int(max_parameter)}B+",
                    elem_id="double-slider-1",
                    step=parameter_step
                )

                
            ########### LOG SLIDER 2 ################

            with gr.Row():
                context_slider = RangeSlider(
                    minimum=0, 
                    maximum=max_context, 
                    label="Select Context Range (k) 📏",
                    elem_id="double-slider-2",
                    step=context_step
                )

            ############# PRICE SLIDER 1 ###############
            with gr.Row():
                input_pricing_slider = RangeSlider(
                    minimum=0, 
                    maximum=max_input_price, 
                    value=(0, max_input_price), 
                    label="Select Price range 💲/1M input tokens",
                    elem_id="double-slider-3"
                )
            
            ############### PRICE SLIDER 2 ###############
            with gr.Row():
                output_pricing_slider = RangeSlider(
                    minimum=0, 
                    maximum=max_output_price, 
                    value=(0, max_output_price), 
                    label="Select Price range 💲/1M output tokens",
                    elem_id="double-slider-4"
                )
       

    with gr.Row():
        """
        Main Leaderboard Row
        """

        leaderboard_table = gr.Dataframe(
                                value=short_leaderboard,
                                elem_id="text-leaderboard-table",
                                interactive=False,
                                visible=True,
                                datatype=['html', 'number', 'number', 'date', 'number', 'number', 'number', 'number', 'html']
                            )

        
        dummy_leaderboard_table = gr.Dataframe(
                                value=text_leaderboard,
                                elem_id="dummy-leaderboard-table",
                                interactive=False,
                                visible=False
                            )
        
        lang_dropdown.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],   
            queue=True
        )

        parameter_slider.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        input_pricing_slider.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        output_pricing_slider.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        multimodal_checkbox.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        open_weight_checkbox.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        context_slider.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        start_date.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        end_date.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

        license_checkbox.change(
            filter,
            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
            [leaderboard_table],
            queue=True
        )

    llm_calc_app.load()
llm_calc_app.queue()
llm_calc_app.launch()



"""
model_name, input_price, output_price,
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
source,licence_name,licence_url,languages,release_date,
parameters_estimated,parameters_actual,

open_weight,context, 

additional_prices_context_caching,
additional_prices_context_storage,
additional_prices_image_input,additional_prices_image_output,additional_prices_video_input,additional_prices_video_output,additional_prices_audio_input,additional_prices_audio_output,clemscore_v1.6.5_multimodal,clemscore_v1.6.5_ascii,clemscore_v1.6,latency_v1.6,latency_v1.6.5_multimodal,latency_v1.6.5_ascii,

average_clemscore,average_latency,parameters

Final list

model_name, input_price, output_price,
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
source,licence_name,licence_url,languages,release_date, open_weight,context, average_clemscore,average_latency,parameters


Filter
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
licence_name+licence_url, languages, release_date, open_weight

RR
model_name, input_price, output_price,
source, release_date

"""