Upload 11 files

README.md

@@ -1,10 +1,10 @@
 ---
-title: TestLLMCalc
-emoji: 📈
-colorFrom: pink
-colorTo: purple
+title: LLMCalc
+emoji: 🏆
+colorFrom: red
+colorTo: pink
 sdk: gradio
-sdk_version: 5.12.0
+sdk_version: 4.44.1
 app_file: app.py
 pinned: false
 ---

app.py

@@ -0,0 +1,330 @@
+import pandas as pd
+import gradio as gr
+import os
+from gradio_rangeslider import RangeSlider
+
+from src.filter_utils import filter, filter_cols
+
+# Main Leaderboard containing everything
+text_leaderboard = pd.read_csv(os.path.join('assets', 'merged_data.csv'))
+text_leaderboard = text_leaderboard.sort_values(by='Clemscore', ascending=False)  
+
+open_weight_df = text_leaderboard[text_leaderboard['Open Weight'] == True]
+if not open_weight_df.empty:  # Check if filtered df is non-empty
+    max_parameter_size = open_weight_df['Parameters (B)'].max()
+
+# Short leaderboard containing fixed columns
+short_leaderboard = filter_cols(text_leaderboard)
+
+## Extract data
+langs = []
+licenses = []
+ip_prices = []
+op_prices = []
+latencies = []
+parameters = []
+contexts = []
+dates = []
+
+for i in range(len(text_leaderboard)):
+    lang_splits = text_leaderboard.iloc[i]['Languages'].split(',')
+    lang_splits = [s.strip() for s in lang_splits]
+    langs += lang_splits
+    license_name = text_leaderboard.iloc[i]['License Name']
+
+    licenses.append(license_name)
+    ip_prices.append(text_leaderboard.iloc[i]['Input $/1M tokens'])
+    op_prices.append(text_leaderboard.iloc[i]['Output $/1M tokens'])
+    latencies.append(text_leaderboard.iloc[i]['Latency (s)'])
+    parameters.append(text_leaderboard.iloc[i]['Parameters (B)'])
+    contexts.append(text_leaderboard.iloc[i]['Context Size (k)'])
+    dates.append(text_leaderboard.iloc[i]['Release Date'])
+
+
+langs = list(set(langs))
+langs.sort()
+
+licenses = list(set(licenses))
+licenses.sort()
+
+max_input_price = max(ip_prices)
+max_output_price = max(op_prices)
+max_latency = max(latencies)
+
+min_parameters = 0 if pd.isna(min(parameters)) else min(parameters)
+max_parameter = max_parameter_size
+parameter_step = 1
+print(f"MIN {min_parameters}, MAX {max_parameter}")
+
+min_context = min(contexts)
+max_context = max(contexts)
+context_step = 8
+
+min_date = min(dates)
+max_date = max(dates)
+
+TITLE = """<h1 align="center" id="space-title"> LLM Calculator ⚖️⚡ 📏💰</h1>"""
+CSS = """
+#double-slider-1 {height: 100px}
+#double-slider-2 {height: 100px}
+#double-slider-3 {height: 100px}
+#double-slider-4 {height: 100px}
+"""
+
+llm_calc_app = gr.Blocks(css=CSS)
+with llm_calc_app:
+
+    gr.HTML(TITLE)
+
+    ##################################################
+
+    with gr.Row():
+
+        #####################################
+        # First Column
+        ####################################
+        ## Language Select
+        with gr.Column():
+
+            with gr.Row():
+                lang_dropdown = gr.Dropdown(
+                    choices=langs,
+                    value=[],
+                    multiselect=True,
+                    label="Select Languages 🗣️"
+                )
+
+            with gr.Row():
+                start_date = gr.DateTime(
+                    value=min_date,
+                    type="string",
+                    label="Release Date Range 📅 - Start Date"
+                )
+
+                end_date = gr.DateTime(
+                    value=max_date,
+                    type="string",
+                    label="End Date"
+                )
+
+            # Multiodality Select
+            with gr.Row():
+                multimodal_checkbox = gr.CheckboxGroup(
+                    choices=['Image', 'Multi-Image', 'Audio', 'Video'],
+                    value=[],
+                    label="Select Additional Modalities 📷🎧🎬",
+                )
+
+            # Open/Commercial Selection
+            with gr.Row():
+                open_weight_checkbox = gr.CheckboxGroup(
+                    choices=['Open', 'Commercial'],
+                    value=['Open', 'Commercial'],
+                    label="Filter by Model Type 🔓 💼",
+                )    
+
+            # License selection
+            with gr.Row():
+                license_checkbox = gr.CheckboxGroup(
+                    choices=licenses,
+                    value=licenses,
+                    label="License Type 🛡️",
+                )    
+        
+        #############################################################
+        # Second Column
+        #############################################################
+        with gr.Column():
+
+            ####### LOG SLIDER 1 ###########
+            with gr.Row():
+                parameter_slider = RangeSlider(
+                    minimum=0, 
+                    maximum=max_parameter, 
+                    label=f"Select Parameter Range 🔍 {int(min_parameters)}B - {int(max_parameter)}B+",
+                    elem_id="double-slider-1",
+                    step=parameter_step
+                )
+
+                
+            ########### LOG SLIDER 2 ################
+
+            with gr.Row():
+                context_slider = RangeSlider(
+                    minimum=0, 
+                    maximum=max_context, 
+                    label="Select Context Range (k) 📏",
+                    elem_id="double-slider-2",
+                    step=context_step
+                )
+
+            ############# PRICE SLIDER 1 ###############
+            with gr.Row():
+                input_pricing_slider = RangeSlider(
+                    minimum=0, 
+                    maximum=max_input_price, 
+                    value=(0, max_input_price), 
+                    label="Select Price range 💲/1M input tokens",
+                    elem_id="double-slider-3"
+                )
+            
+            ############### PRICE SLIDER 2 ###############
+            with gr.Row():
+                output_pricing_slider = RangeSlider(
+                    minimum=0, 
+                    maximum=max_output_price, 
+                    value=(0, max_output_price), 
+                    label="Select Price range 💲/1M output tokens",
+                    elem_id="double-slider-4"
+                )
+       
+
+    with gr.Row():
+        """
+        Main Leaderboard Row
+        """
+
+        leaderboard_table = gr.Dataframe(
+                                value=short_leaderboard,
+                                elem_id="text-leaderboard-table",
+                                interactive=False,
+                                visible=True,
+                                datatype=['html', 'number', 'number', 'date', 'number', 'number', 'number', 'number', 'html']
+                            )
+
+        
+        dummy_leaderboard_table = gr.Dataframe(
+                                value=text_leaderboard,
+                                elem_id="dummy-leaderboard-table",
+                                interactive=False,
+                                visible=False
+                            )
+        
+        lang_dropdown.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],   
+            queue=True
+        )
+
+        parameter_slider.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        input_pricing_slider.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        output_pricing_slider.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        multimodal_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        open_weight_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        context_slider.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        start_date.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        end_date.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        license_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+    llm_calc_app.load()
+llm_calc_app.queue()
+llm_calc_app.launch()
+
+
+
+"""
+model_name, input_price, output_price,
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+source,licence_name,licence_url,languages,release_date,
+parameters_estimated,parameters_actual,
+
+open_weight,context, 
+
+additional_prices_context_caching,
+additional_prices_context_storage,
+additional_prices_image_input,additional_prices_image_output,additional_prices_video_input,additional_prices_video_output,additional_prices_audio_input,additional_prices_audio_output,clemscore_v1.6.5_multimodal,clemscore_v1.6.5_ascii,clemscore_v1.6,latency_v1.6,latency_v1.6.5_multimodal,latency_v1.6.5_ascii,
+
+average_clemscore,average_latency,parameters
+
+Final list
+
+model_name, input_price, output_price,
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+source,licence_name,licence_url,languages,release_date, open_weight,context, average_clemscore,average_latency,parameters
+
+
+Filter
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+licence_name+licence_url, languages, release_date, open_weight
+
+RR
+model_name, input_price, output_price,
+source, release_date
+
+"""
+
+

assets/merged_data.csv

@@ -0,0 +1,71 @@
+Model Name,Latency (s),Clemscore,Parameters (B),Release Date,Open Weight,Languages,Context Size (k),License Name,License URL,Single Image,Multiple Images,Audio,Video,Input $/1M tokens,Output $/1M tokens,License,Temp Date
+o1-preview-2024-09-12,7.368572853601854,73.63,,2024-09-12,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,15.0,60.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-09-12
+gpt-4-1106-vision-preview,4.712557435752081,73.55,,2023-11-06,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,10.0,30.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-06
+claude-3-5-sonnet-20240620,2.0645066812060726,68.925,,2024-06-20,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,3.0,15.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-06-20
+gpt-4o-2024-08-06,1.951333607454077,63.875,,2024-08-06,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,3.75,15.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-08-06
+gpt-4o-2024-05-13,5.022646224034688,58.95,,2024-05-13,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,5.0,15.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-05-13
+gpt-4-turbo-2024-04-09,,58.3,,2024-04-09,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,10.0,30.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-04-09
+claude-3-opus-20240229,3.916101346449241,55.29,,2024-02-29,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,15.0,75.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-02-29
+gpt-4-0125-preview,1.0418927523113648,52.5,,2024-01-25,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,10.0,30.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-01-25
+Meta-Llama-3.1-405B-Instruct-Turbo,0.7886103946545819,52.11,405.0,2024-07-23,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-23
+gpt-4-1106-preview,0.7767265743542736,51.99,,2023-11-06,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-06
+gpt-4-0613,0.648441146582876,51.09,,2023-06-13,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-06-13
+gpt-4o-mini-2024-07-18,2.08647007916325,46.55,,2024-07-18,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.3,1.2,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-18
+Mistral-Large-Instruct-2407,1.2444667688634192,45.39,123.0,2024-07-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-24
+Meta-Llama-3.1-70B-Instruct,0.8105055275945292,38.83,70.0,2024-07-23,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-23
+InternVL2-26B,4.239272214812438,37.45,26.0,2024-07-15,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-15
+InternVL2-Llama3-76B,10.660117299385416,33.84,76.0,2024-07-15,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-15
+claude-2.1,1.6836316221022516,32.5,,2023-11-21,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,8.0,24.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-21
+InternVL2-40B,6.267102418391484,32.23,40.0,2024-07-15,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-15
+claude-3-sonnet-20240229,1.4194860128225952,30.53,,2024-02-29,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,3.0,15.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-02-29
+Qwen1.5-72B-Chat,12.689668927658191,30.37,72.0,2024-01-30,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-01-30
+Qwen2-72B-Instruct,0.9480584860151366,30.03,72.0,2024-05-28,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-05-28
+idefics-80b-instruct,6.8089303915502315,29.55,80.0,2023-07-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-07-24
+Pixtral-12B-2409,1.4976731684122335,28.64,12.0,2024-09-11,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-09-11
+mistral-large-2402,0.3967416598893965,28.17,123.0,2024-02-01,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-02-01
+Qwen2.5-Coder-32B-Instruct,0.8337066960552915,27.57,32.0,2024-11-06,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-11-06
+gemma-2-9b-it,0.3692553324432573,27.34,9.0,2024-06-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-06-24
+gpt-3.5-turbo-0125,,27.22,,2024-01-25,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.5,1.5,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-01-25
+command-r-plus,0.3104016019283746,24.94,104.0,2024-04-01,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-04-01
+openchat_3.5,0.3172876868462049,23.64,7.0,2023-10-30,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-10-30
+InternVL2-8B,1.948600327851168,23.17,8.0,2024-07-15,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-15
+claude-3-haiku-20240307,0.8695497396191068,22.49,,2024-03-07,False,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.25,1.25,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-03-07
+sheep-duck-llama-2-70b-v1.1,5.524607914346901,21.5,70.0,2023-09-27,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-09-27
+Meta-Llama-3.1-8B-Instruct,0.206305748406081,18.36,8.0,2024-07-23,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-23
+openchat-3.5-1210,0.280498276910299,18.22,7.0,2023-12-10,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-12-10
+Idefics3-8B-Llama3,2.7247848158020003,17.52,8.0,2024-08-05,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-08-05
+WizardLM-70b-v1.0,3.924977203883497,17.4,70.0,2023-08-09,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-08-09
+openchat-3.5-0106,0.2920951450556648,17.1,7.0,2024-01-06,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-01-06
+internlm-xcomposer2d5-7b,8.438096179522176,16.95,7.0,2024-07-02,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-02
+mistral-medium-2312,3.3167870515212083,16.43,,2023-12-01,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-12-01
+Phi-3.5-vision-instruct,1.540488050470713,15.64,4.0,2024-08-17,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-08-17
+codegemma-7b-it,0.3048974050865229,15.3,7.0,2024-04-09,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-04-09
+CodeLlama-34b-Instruct-hf,3.851887315425933,14.35,34.0,2023-08-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-08-24
+command-r,0.1883241491458606,14.15,35.0,2024-03-01,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-03-01
+gemma-1.1-7b-it,0.1782953878345496,14.14,7.0,2024-03-26,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-03-26
+SUS-Chat-34B,2.27951476106911,14.11,34.0,2023-11-29,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-29
+aya-23-35B,0.5755088395104287,13.35,35.0,2024-05-19,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-05-19
+Mixtral-8x22B-Instruct-v0.1,1.0759354563573875,12.69,141.0,2024-04-17,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-04-17
+tulu-2-dpo-70b,7.848597339328536,12.62,70.0,2023-11-12,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-12
+idefics-9b-instruct,4.156911970172687,12.29,9.0,2023-07-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-07-24
+aya-23-8B,0.4818848185613353,11.72,8.0,2024-05-19,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-05-19
+WizardLM-13b-v1.2,3.5654367625763,11.48,13.0,2023-07-25,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-07-25
+vicuna-33b-v1.3,0.8235025152162306,11.27,33.0,2023-06-21,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-06-21
+Llama-3.1-Nemotron-70B-Instruct-HF,1.105406813859938,10.16,70.0,2024-10-12,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-10-12
+Yi-34B-Chat,1.2871676207135438,8.27,34.0,2023-11-22,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-11-22
+Mixtral-8x7B-Instruct-v0.1,0.9392967660636314,8.17,46.7,2023-12-11,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-12-11
+Mixtral-8x7B-Instruct-v0.1,0.9392967660636314,8.17,46.7,2023-12-11,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-12-11
+Mistral-7B-Instruct-v0.1,0.2828647550771728,8.01,7.0,2023-09-27,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-09-27
+vicuna-13b-v1.5,1.4753938719676598,7.01,13.0,2023-07-29,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-07-29
+Starling-LM-7B-beta,1.365002297029703,6.56,7.0,2024-03-19,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-03-19
+Phi-3-mini-128k-instruct,0.6615315832127354,6.33,3.8,2024-04-22,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-04-22
+Qwen2-7B-Instruct,0.3589407217948714,6.18,7.0,2024-06-04,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-06-04
+salamandra-7b-instruct,0.3894831193548387,6.04,7.0,2024-09-30,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-09-30
+sheep-duck-llama-2-13b,2.9462099794520573,5.39,13.0,2023-10-04,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-10-04
+dolphin-vision-72b,10.190958003739729,4.65,72.0,2024-06-28,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,True,True,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-06-28
+gemma-2-27b-it,0.9922771009345794,3.51,27.0,2024-06-24,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-06-24
+gemma-1.1-2b-it,0.1192569946127946,2.91,2.0,2024-03-26,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-03-26
+gemma-2-2b-it,0.3139821517919889,2.67,2.0,2024-07-16,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-07-16
+Qwen1.5-7B-Chat,0.3898907690883847,2.58,7.0,2024-01-30,True,"Chinese, English",,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-01-30
+gemma-7b-it,0.6112263564356434,1.82,7.0,2024-02-21,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2024-02-21
+llama-2-70b-chat-hf,4.724659620079607,0.81,70.0,2023-07-18,True,English,,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,False,False,False,False,0.0,0.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",2023-07-18

assets/pricing.json

@@ -0,0 +1,212 @@
+[
+    {
+        "model_id": "gpt-4-1106-vision-preview",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "gpt-4o-2024-05-13",
+        "input": "5$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gpt-4o-2024-08-06",
+        "input": "3.750$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gpt-4o-mini-2024-07-18",
+        "input": "0.300$",
+        "output": "1.200$"
+    },
+    {
+        "model_id": "gpt-4-turbo-2024-04-09",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "gpt-4-1106-preview",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-4-0125-preview",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "o1-preview-2024-09-12",
+        "input": "15$",
+        "output": "60$"
+    },
+    {
+        "model_id": "o1-mini-2024-09-12",
+        "input": "3$",
+        "output": "12$"
+    },
+    {
+        "model_id": "gpt-3.5-turbo-0125",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gpt-4-0613",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-4-0314",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-3.5-turbo-1106",
+        "input": "1$",
+        "output": "2$"
+    },
+    {
+        "model_id": "gpt-3.5-turbo-0613",
+        "input": "1.5$",
+        "output": "2$"
+    },
+    {
+        "model_id": "command",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "command-light",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-v1.3",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-v1.3-100k",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-instant-1.2",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-2",
+        "input": "8$",
+        "output": "24$"
+    },
+    {
+        "model_id": "claude-2.1",
+        "input": "8$",
+        "output": "24$"
+    },
+    {
+        "model_id": "claude-3-opus-20240229",
+        "input": "15$",
+        "output": "75$"
+    },
+    {
+        "model_id": "claude-3-sonnet-20240229",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "claude-3-haiku-20240307",
+        "input": "0.25$",
+        "output": "1.25$"
+    },
+    {
+        "model_id": "claude-3-5-sonnet-20240620",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "claude-3-5-haiku-20241022",
+        "input": "0.8$",
+        "output": "4$"
+    },
+    {
+        "model_id": "claude-3-5-sonnet-20241022",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-001",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-002",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-vision-latest",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-001",
+        "input": "0.075$",
+        "output": "0.3$"
+    },
+    {
+        "model_id": "gemini-1.5-pro-001",
+        "input": "1.25$",
+        "output": "5$"
+    },
+    {
+        "model_id": "gemini-1.5-pro-002",
+        "input": "1.25$",
+        "output": "5$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-002",
+        "input": "0.075$",
+        "output": "0.3$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-8b-001",
+        "input": "0.0375$",
+        "output": "0.15$"
+    },
+    {
+        "model_id": "gemini-2.0-flash-exp",
+        "input": "0$",
+        "output": "0$"
+    },
+    {
+        "model_id": "luminous-supreme-control",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-supreme",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-extended",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    }
+]

assets/text_content.py

@@ -0,0 +1,53 @@
+import os 
+
+CLEMBENCH_RUNS_REPO = "https://raw.githubusercontent.com/clembench/clembench-runs/main/"
+REGISTRY_URL = "https://raw.githubusercontent.com/clp-research/clembench/refs/heads/refactor_model_registry/backends/model_registry.json"
+BENCHMARK_FILE = "benchmark_runs.json"
+
+LATENCY_FOLDER = os.path.join("Addenda", "Latency")
+RESULT_FILE = "results.csv"
+LATENCY_SUFFIX = "_latency.csv"
+
+LANG_MAPPING = {
+    'el': 'Greek',
+    'id': 'Indonesian',
+    'ko': 'Korean',
+    'sv': 'Swedish',
+    'de': 'German',
+    'lv': 'Latvian',
+    'am': 'Amharic',
+    'fi': 'Finnish',
+    'da': 'Danish',
+    'pt': 'Portuguese',
+    'sw': 'Swahili',
+    'es': 'Spanish',
+    'it': 'Italian',
+    'bn': 'Bengali',
+    'nl': 'Dutch',
+    'lt': 'Lithuanian',
+    'ro': 'Romanian',
+    'sl': 'Slovenian',
+    'hu': 'Hungarian',
+    'hr': 'Croatian',
+    'vi': 'Vietnamese',
+    'hi': 'Hindi',
+    'zh': 'Chinese',
+    'pl': 'Polish',
+    'ar': 'Arabic',
+    'cs': 'Czech',
+    'sk': 'Slovak',
+    'ja': 'Japanese',
+    'no': 'Norwegian',
+    'uk': 'Ukrainian',
+    'fr': 'French',
+    'et': 'Estonian',
+    'ru': 'Russian',
+    'th': 'Thai',
+    'bg': 'Bulgarian',
+    'tr': 'Turkish',
+    'ms': 'Malay',
+    'he': 'Hebrew',
+    'tl': 'Tagalog',
+    'sr': 'Serbian',
+    'en': 'English'
+}

prepare_path.sh

@@ -0,0 +1,2 @@
+#!/bash/bin
+export PYTHONPATH=.:$PYTHONPATH

requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4==4.12.3
+pandas==2.2.3
+gradio_rangeslider==0.0.7
+gradio==4.44.1
+langcodes==3.5.0

src/collect_data.py

@@ -0,0 +1,152 @@
+"""
+Collect data from the multiple sources and create a base datafranme for the LLMCalculator table
+Latency - https://github.com/clembench/clembench-runs/tree/main/Addenda/Latency
+Pricing - pricing.json
+Model info - https://github.com/kushal-10/clembench/blob/feat/registry/backends/model_registry_updated.json
+"""
+
+import pandas as pd
+import json
+import requests
+from assets.text_content import CLEMBENCH_RUNS_REPO, REGISTRY_URL, BENCHMARK_FILE, LATENCY_FOLDER, RESULT_FILE, LATENCY_SUFFIX
+import os
+
+def validate_request(url: str, response) -> bool:
+    """
+    Validate if an HTTP request was successful.
+    
+    Args:
+        url (str): The URL that was requested
+        response (requests.Response): The response object from the request
+        
+    Returns:
+        bool: True if request was successful (status code 200), False otherwise
+    """
+
+    if response.status_code != 200:
+        print(f"Failed to read file - {url}. Status Code: {response.status_code}")
+        return False
+    return True
+
+def fetch_benchmark_data(benchmark: str = "text", version_names: list = []) -> tuple:
+    """
+    Fetch and parse benchmark results and latency data from CSV files.
+    
+    Args:
+        benchmark (str): Type of benchmark to fetch ('text' or 'multimodal')
+        version_names (list): List of version names to search through, sorted by latest first
+        
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame]: A tuple containing:
+            - results_df: DataFrame with benchmark results
+            - latency_df: DataFrame with latency measurements
+            Returns (None, None) if no matching version is found or requests fail
+            
+    Raises:
+        requests.RequestException: If there's an error fetching the data
+        pd.errors.EmptyDataError: If CSV file is empty
+        pd.errors.ParserError: If CSV parsing fails
+    """
+    for v in version_names:
+        # Check if version matches benchmark type
+        is_multimodal = 'multimodal' in v
+        if (benchmark == "multimodal") != is_multimodal:
+            continue
+            
+        # Construct URLs
+        results_url = os.path.join(CLEMBENCH_RUNS_REPO, v, RESULT_FILE)
+        latency_url = os.path.join(CLEMBENCH_RUNS_REPO, LATENCY_FOLDER, v + LATENCY_SUFFIX)
+        
+        try:
+            results = requests.get(results_url)
+            latency = requests.get(latency_url)
+            
+            if validate_request(results_url, results) and validate_request(latency_url, latency):
+                # Convert the CSV content to pandas DataFrames
+                results_df = pd.read_csv(pd.io.common.StringIO(results.text))
+                latency_df = pd.read_csv(pd.io.common.StringIO(latency.text))
+                return results_df, latency_df
+                
+        except requests.RequestException as e:
+            print(f"Error fetching data for version {v}: {e}")
+        except pd.errors.EmptyDataError:
+            print(f"Error: Empty CSV file found for version {v}")
+        except pd.errors.ParserError:
+            print(f"Error: Unable to parse CSV data for version {v}")
+            
+    return None, None
+
+def fetch_version_metadata() -> tuple:
+    """
+    Fetch and process benchmark metadata from the Clembench GitHub repository.
+    
+    The data is sourced from: https://github.com/clembench/clembench-runs
+    Configure the repository path in src/assets/text_content/CLEMBENCH_RUNS_REPO
+    
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: A tuple containing:
+            - mm_result: Multimodal benchmark results
+            - mm_latency: Multimodal latency data
+            - text_result: Text benchmark results
+            - text_latency: Text latency data
+            Returns (None, None, None, None) if the request fails
+    """
+    json_url = CLEMBENCH_RUNS_REPO + BENCHMARK_FILE
+    response = requests.get(json_url)
+
+    # Check if the JSON file request was successful
+    if not validate_request(json_url, response):
+        return None, None, None, None
+
+    json_data = response.json()
+    versions = json_data['versions']
+
+    # Sort the versions in benchmark by latest first
+    version_names = sorted(
+        [ver['version'] for ver in versions],
+        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  
+        reverse=True
+    )
+
+    # Latency is in  seconds
+    mm_result, mm_latency = fetch_benchmark_data("multimodal", version_names)
+    text_result, text_latency = fetch_benchmark_data("text", version_names)
+
+    return mm_latency, mm_result, text_latency, text_result
+
+def fetch_registry_data() -> dict:
+    """
+    Fetch and parse model registry data from the Clembench registry URL.
+    
+    The data is sourced from the model registry defined in REGISTRY_URL.
+    Contains information about various LLM models including their specifications
+    and capabilities.
+    
+    Returns:
+        dict: Dictionary containing model registry data.
+        Returns None if the request fails or the JSON is invalid.
+        
+    Raises:
+        requests.RequestException: If there's an error fetching the data
+        json.JSONDecodeError: If the response cannot be parsed as JSON
+    """
+    try:
+        response = requests.get(REGISTRY_URL)
+        if not validate_request(REGISTRY_URL, response):
+            return None
+            
+        return response.json()
+        
+    except requests.RequestException as e:
+        print(f"Error fetching registry data: {e}")
+    except json.JSONDecodeError as e:
+        print(f"Error parsing registry JSON: {e}")
+    
+    return None
+
+if __name__=="__main__":
+    fetch_version_metadata()
+    registry_data = fetch_registry_data()
+    print(registry_data[0])
+
+    

src/filter_utils.py

@@ -0,0 +1,109 @@
+# Utility functions for filtering the dataframe
+
+import pandas as pd
+
+def filter_cols(df):
+
+    df = df[[
+    'Model Name', 
+    'Clemscore',
+    'Input $/1M tokens', 
+    'Output $/1M tokens',
+    'Latency (s)',
+    'Context Size (k)', 
+    'Parameters (B)',
+    'Release Date', 
+    'License'
+    ]]
+    
+    return df
+
+
+def filter(df, language_list, parameters, input_price, output_price, multimodal,
+           context, open_weight, start, end, license ):
+    
+
+    if not df.empty:  # Check if df is non-empty
+        df = df[df['Languages'].apply(lambda x: all(lang in x for lang in language_list))]
+
+    if not df.empty:
+        # Split dataframe by Open Weight
+        open_weight_true = df[df['Open Weight'] == True]
+        open_weight_false = df[df['Open Weight'] == False]
+        
+        # Get max parameter size for open weight models
+        max_parameter_size = open_weight_true['Parameters (B)'].max() if not open_weight_true.empty else 0
+        
+        # Filter only the open weight models based on parameters
+        if not open_weight_true.empty:
+            if parameters[1] >= max_parameter_size:
+                filtered_open = open_weight_true[
+                    (open_weight_true['Parameters (B)'] >= parameters[0])
+                ]
+            else:
+                filtered_open = open_weight_true[
+                    (open_weight_true['Parameters (B)'] >= parameters[0]) & 
+                    (open_weight_true['Parameters (B)'] <= parameters[1])
+                ]
+            
+            # Combine filtered open weight models with unfiltered commercial models
+            df = pd.concat([filtered_open, open_weight_false])
+
+    if not df.empty:  # Check if df is non-empty
+        df = df[(df['Input $/1M tokens'] >= input_price[0]) & (df['Input $/1M tokens'] <= input_price[1])]
+    
+    if not df.empty:  # Check if df is non-empty
+        df = df[(df['Output $/1M tokens'] >= output_price[0]) & (df['Output $/1M tokens'] <= output_price[1])]
+
+
+    print("Price")    
+    print(df)
+
+    if not df.empty:  # Check if df is non-empty
+        if "Image" in multimodal:
+            df = df[df['Image'] == True]
+        if "Multi-Image" in multimodal:
+            df = df[df['Multiple Image'] == True]
+        if "Audio" in multimodal:
+            df = df[df['Audio'] == True]
+        if "Video" in multimodal:
+            df = df[df['Video'] == True]
+
+    # if not df.empty:  # Check if df is non-empty
+    #     df = df[(df['Context Size (k)'] >= (context[0])) & (df['Context Size (k)'] <= (context[1]))]
+
+
+    print("Modality")    
+    print(df)
+
+    if not df.empty:  # Check if df is non-empty
+        if "Open" in open_weight and "Commercial" not in open_weight:
+            df = df[df['Open Weight'] == True]
+        elif "Commercial" in open_weight and "Open" not in open_weight:
+            df = df[df['Open Weight'] == False]
+        elif "Open" not in open_weight and "Commercial" not in open_weight:
+            # Return empty DataFrame with same columns
+            df = pd.DataFrame(columns=df.columns)
+        
+    if not df.empty:  # Check if df is non-empty
+        df = df[df['License Name'].apply(lambda x: any(lic in x for lic in license))]
+
+    # Convert 'Release Date' to int temporarily
+    if not df.empty:  # Check if df is non-empty
+        df['Temp Date'] = pd.to_datetime(df['Temp Date']).astype(int) // 10**9  # Convert to seconds since epoch
+
+    # Convert start and end to int (seconds since epoch)
+    start = int(pd.to_datetime(start).timestamp())  
+    end = int(pd.to_datetime(end).timestamp())    
+
+    # Filter based on the converted 'Release Date'
+    if not df.empty:  # Check if df is non-empty
+        df = df[(df['Temp Date'] >= start) & (df['Temp Date'] <= end)]
+
+    df = filter_cols(df)
+    df = df.sort_values(by='Clemscore', ascending=False)
+
+    print(df)
+
+    return df  # Return the filtered dataframe
+    

src/process_data.py

@@ -0,0 +1,163 @@
+import pandas as pd
+import json
+import os
+
+from src.collect_data import fetch_version_metadata, fetch_registry_data
+from assets.text_content import LANG_MAPPING
+PRICING_PATH = os.path.join('assets', 'pricing.json')
+
+# Convert parameters to float, handling both B and T suffixes
+def convert_parameters(param):
+    if pd.isna(param) or param == '':
+        return None
+    param = str(param)
+    if 'T' in param:
+        return float(param.replace('T', '')) * 1000
+    return float(param.replace('B', ''))
+
+# Clean price strings by removing '$' and handling empty strings
+def clean_price(price):
+    if pd.isna(price) or price == '':
+        return None
+    return float(price.replace('$', ''))
+
+# Handle language mapping for both string and list inputs
+def map_languages(languages):
+    if isinstance(languages, float) and pd.isna(languages):
+        return None
+    # If it's already a list
+    if isinstance(languages, list):
+        return ', '.join([LANG_MAPPING.get(str(lang), str(lang)) for lang in languages])
+    # If it's a string
+    if isinstance(languages, str):
+        return ', '.join([LANG_MAPPING.get(lang.strip(), lang.strip()) for lang in languages.split(',')])
+    # If it's an array or any other type
+    try:
+        return ', '.join([str(lang) for lang in languages])
+    except:
+        return str(languages)
+    
+# Extract multimodality fields
+def get_multimodality_field(model_data, field):
+    try:
+        return model_data.get('model_config', {}).get('multimodality', {}).get(field, False)
+    except:
+        return False
+
+
+def merge_data():
+
+    mm_latency_df, mm_result_df, text_latency_df, text_result_df = fetch_version_metadata()
+    registry_data = fetch_registry_data()
+    with open(PRICING_PATH, 'r') as f:
+        pricing_data = json.load(f)
+
+    # Ensure the unnamed column is renamed to 'model'
+    mm_result_df.rename(columns={'Unnamed: 0': 'model', '-, clemscore': 'clemscore'}, inplace=True)
+    text_result_df.rename(columns={'Unnamed: 0': 'model', '-, clemscore': 'clemscore'}, inplace=True)
+    mm_result_df['model'] = mm_result_df['model'].str.split('-t0.0--').str[0]
+    text_result_df['model'] = text_result_df['model'].str.split('-t0.0--').str[0] # Bug in get_latency.py, split by -t0.0 instead of -t (gpt-3.5-turbo/gpt-4-turbo breaks) 
+
+    # Merge datasets to compute average values
+    avg_latency_df = pd.concat([mm_latency_df, text_latency_df], axis=0).groupby('model')['latency'].mean().reset_index()
+    avg_clemscore_df = pd.concat([mm_result_df, text_result_df], axis=0).groupby('model')['clemscore'].mean().reset_index()
+
+    # Merge latency, clemscore, registry, and pricing data
+    lat_clem_df = pd.merge(avg_latency_df, avg_clemscore_df, on='model', how='outer')
+
+    # Convert registry_data to DataFrame for easier merging
+    registry_df = pd.DataFrame(registry_data)
+    
+    # Extract license info
+    registry_df['license_name'] = registry_df['license'].apply(lambda x: x['name'])
+    registry_df['license_url'] = registry_df['license'].apply(lambda x: x['url'])
+
+    # Add individual multimodality columns
+    registry_df['single_image'] = registry_df.apply(lambda x: get_multimodality_field(x, 'single_image'), axis=1)
+    registry_df['multiple_images'] = registry_df.apply(lambda x: get_multimodality_field(x, 'multiple_images'), axis=1)
+    registry_df['audio'] = registry_df.apply(lambda x: get_multimodality_field(x, 'audio'), axis=1)
+    registry_df['video'] = registry_df.apply(lambda x: get_multimodality_field(x, 'video'), axis=1)
+
+    # Update columns list to include new multimodality fields
+    registry_df = registry_df[[
+        'model_name', 'parameters', 'release_date', 'open_weight',
+        'languages', 'context_size', 'license_name', 'license_url',
+        'single_image', 'multiple_images', 'audio', 'video'
+    ]]
+    
+    # Merge with previous data
+    merged_df = pd.merge(
+        lat_clem_df,
+        registry_df,
+        left_on='model',
+        right_on='model_name',
+        how='inner'
+    )
+    
+    # Update column renaming
+    merged_df = merged_df.rename(columns={
+        'model': 'Model Name',
+        'latency': 'Latency (s)',
+        'clemscore': 'Clemscore',
+        'parameters': 'Parameters (B)',
+        'release_date': 'Release Date',
+        'open_weight': 'Open Weight',
+        'languages': 'Languages',
+        'context_size': 'Context Size (k)',
+        'license_name': 'License Name',
+        'license_url': 'License URL',
+        'single_image': 'Single Image',
+        'multiple_images': 'Multiple Images',
+        'audio': 'Audio',
+        'video': 'Video'
+    })
+    
+    # Convert pricing_data list to DataFrame
+    pricing_df = pd.DataFrame(pricing_data)
+    pricing_df['input'] = pricing_df['input'].apply(clean_price)
+    pricing_df['output'] = pricing_df['output'].apply(clean_price)
+    
+    # Merge pricing data with the existing dataframe
+    merged_df = pd.merge(
+        merged_df,
+        pricing_df,
+        left_on='Model Name',
+        right_on='model_id',
+        how='left'
+    )
+    
+    # Drop duplicate model column and rename price columns
+    merged_df = merged_df.drop('model_id', axis=1)
+    merged_df = merged_df.rename(columns={
+        'input': 'Input $/1M tokens',
+        'output': 'Output $/1M tokens'
+    })
+    
+    # Fill NaN values with 0.0 for pricing columns
+    merged_df['Input $/1M tokens'] = merged_df['Input $/1M tokens'].fillna(0.0)
+    merged_df['Output $/1M tokens'] = merged_df['Output $/1M tokens'].fillna(0.0)
+    
+    # Convert parameters and set to None for commercial models
+    merged_df['Parameters (B)'] = merged_df.apply(
+        lambda row: None if not row['Open Weight'] else convert_parameters(row['Parameters (B)']), 
+        axis=1
+    )
+
+    merged_df['License'] = merged_df.apply(lambda row: f'<a href="{row["License URL"]}" style="color: blue;">{row["License Name"]}</a>', axis=1)
+    merged_df['Temp Date'] = merged_df['Release Date']
+
+    merged_df['Languages'] = merged_df['Languages'].apply(map_languages)
+
+    # Sort by Clemscore in descending order
+    merged_df = merged_df.sort_values(by='Clemscore', ascending=False)
+    
+    # Drop model_name column
+    merged_df.drop(columns=['model_name'], inplace=True)
+    
+    return merged_df
+
+if __name__=='__main__':
+    merged_df = merge_data()
+    # # Save to CSV
+    output_path = os.path.join('assets', 'merged_data.csv')
+    merged_df.to_csv(output_path, index=False)    

tempapp.py

@@ -0,0 +1,33 @@
+import gradio as gr
+from date_rangeslider import RangeSlider
+from pathlib import Path
+
+text = "## The selected date range is: {min} to {max}"
+
+with gr.Blocks() as demo:
+    with gr.Tabs():
+        with gr.Tab("Demo"):
+            gr.Markdown("""## 🛝 Date RangeSlider
+
+            ## Drag either end and see the selected date range update in real-time.
+            """) 
+            range_slider = RangeSlider(
+                minimum="2023-01-01",
+                maximum="2024-12-31",
+                value=("2023-01-01", "2024-12-31")
+            )
+            range_ = gr.Markdown(value=text.format(min="2023-01-01", max="2024-12-31"))
+            range_slider.change(
+                lambda s: text.format(min=s[0], max=s[1]), 
+                range_slider, 
+                range_,
+                show_progress="hide", 
+                trigger_mode="always_last"
+            )
+            gr.Examples([
+                ("2023-03-01", "2023-06-30"), 
+                ("2023-07-01", "2023-12-31")
+            ], inputs=[range_slider])
+
+if __name__ == "__main__":
+    demo.launch()