Update app.py

app.py

@@ -1,334 +1,145 @@
+import streamlit as st
+import pandas as pd
 import io
 import re
-from collections.abc import Iterable
-
-import pandas as pd
-import streamlit as st
-from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_numeric_dtype
 
-GITHUB_URL = "https://github.com/msamwelmollel/Swahili_LLM_Leaderboard"
-NON_BENCHMARK_COLS = ["Open?", "Publisher"]
-
-
-def extract_table_and_format_from_markdown_text(markdown_table: str) -> pd.DataFrame:
-    """Extracts a table from a markdown text and formats it as a pandas DataFrame.
-    Args:
-        text (str): Markdown text containing a table.
-    Returns:
-        pd.DataFrame: Table as pandas DataFrame.
-    """
-    df = (
-        pd.read_table(io.StringIO(markdown_table), sep="|", header=0, index_col=1)
-        .dropna(axis=1, how="all")  # drop empty columns
-        .iloc[1:]  # drop first row which is the "----" separator of the original markdown table
-        .sort_index(ascending=True)
-        .apply(lambda x: x.str.strip() if x.dtype == "object" else x)
-        .replace("", float("NaN"))
-        .astype(float, errors="ignore")
-    )
-
-    # remove whitespace from column names and index
+# Constants
+GITHUB_URL = "https://github.com/Sartify/STEL"
+NON_BENCHMARK_COLS = ["Open?", "Publisher", "Basemodel", "Matryoshka", "Dimension"]
+
+def extract_table_from_markdown(markdown_text, table_start):
+    """Extract table content from markdown text."""
+    lines = markdown_text.split('\n')
+    table_content = []
+    capture = False
+    for line in lines:
+        if line.startswith(table_start):
+            capture = True
+            continue
+        if capture and line.strip() == '':
+            break
+        if capture:
+            table_content.append(line)
+    return '\n'.join(table_content)
+
+def markdown_table_to_df(table_content):
+    """Convert markdown table to pandas DataFrame."""
+    df = pd.read_csv(io.StringIO(table_content), sep='|', skipinitialspace=True)
     df.columns = df.columns.str.strip()
-    df.index = df.index.str.strip()
-    df.index.name = df.index.name.strip()
-
-    return df
-
-
-def extract_markdown_table_from_multiline(multiline: str, table_headline: str, next_headline_start: str = "#") -> str:
-    """Extracts the markdown table from a multiline string.
-    Args:
-        multiline (str): content of README.md file.
-        table_headline (str): Headline of the table in the README.md file.
-        next_headline_start (str, optional): Start of the next headline. Defaults to "#".
-    Returns:
-        str: Markdown table.
-    Raises:
-        ValueError: If the table could not be found.
-    """
-    # extract everything between the table headline and the next headline
-    table = []
-    start = False
-    for line in multiline.split("\n"):
-        if line.startswith(table_headline):
-            start = True
-        elif line.startswith(next_headline_start):
-            start = False
-        elif start:
-            table.append(line + "\n")
-
-    if len(table) == 0:
-        raise ValueError(f"Could not find table with headline '{table_headline}'")
-
-    return "".join(table)
-
-
-def remove_markdown_links(text: str) -> str:
-    """Modifies a markdown text to remove all markdown links.
-    Example: [DISPLAY](LINK) to DISPLAY
-    First find all markdown links with regex.
-    Then replace them with: $1
-    Args:
-        text (str): Markdown text containing markdown links
-    Returns:
-        str: Markdown text without markdown links.
-    """
-
-    # find all markdown links
-    markdown_links = re.findall(r"\[([^\]]+)\]\(([^)]+)\)", text)
-
-    # remove link keep display text
-    for display, link in markdown_links:
-        text = text.replace(f"[{display}]({link})", display)
-
-    return text
-
-
-def filter_dataframe_by_row_and_columns(df: pd.DataFrame, ignore_columns: list[str] | None = None) -> pd.DataFrame:
-    """
-    Filter dataframe by the rows and columns to display.
-    This does not select based on the values in the dataframe, but rather on the index and columns.
-    Modified from https://blog.streamlit.io/auto-generate-a-dataframe-filtering-ui-in-streamlit-with-filter_dataframe/
-    Args:
-        df (pd.DataFrame): Original dataframe
-        ignore_columns (list[str], optional): Columns to ignore. Defaults to None.
-    Returns:
-        pd.DataFrame: Filtered dataframe
-    """
-    df = df.copy()
-
-    if ignore_columns is None:
-        ignore_columns = []
-
-    modification_container = st.container()
-
-    with modification_container:
-        to_filter_index = st.multiselect("Filter by model:", sorted(df.index))
-        if to_filter_index:
-            df = pd.DataFrame(df.loc[to_filter_index])
-
-        to_filter_columns = st.multiselect(
-            "Filter by benchmark:", sorted([c for c in df.columns if c not in ignore_columns])
-        )
-        if to_filter_columns:
-            df = pd.DataFrame(df[ignore_columns + to_filter_columns])
-
+    df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x)
+    df = df.dropna(axis=1, how='all')
     return df
 
+def setup_page():
+    """Set up the Streamlit page."""
+    st.set_page_config(page_title="Swahili Text Embeddings Leaderboard", page_icon="⚡", layout="wide")
+    st.title("⚡ Swahili Text Embeddings Leaderboard (STEL)")
+    st.image("STEL.jpg", width=300)
 
-def filter_dataframe_by_column_values(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Filter dataframe by the values in the dataframe.
-    Modified from https://blog.streamlit.io/auto-generate-a-dataframe-filtering-ui-in-streamlit-with-filter_dataframe/
-    Args:
-        df (pd.DataFrame): Original dataframe
-    Returns:
-        pd.DataFrame: Filtered dataframe
-    """
-    df = df.copy()
-
-    modification_container = st.container()
-
-    with modification_container:
-        to_filter_columns = st.multiselect("Filter results on:", df.columns)
-        left, right = st.columns((1, 20))
-
-        for column in to_filter_columns:
-            if is_bool_dtype(df[column]):
-                user_bool_input = right.checkbox(f"{column}", value=True)
-                df = df[df[column] == user_bool_input]
-
-            elif is_numeric_dtype(df[column]):
-                _min = float(df[column].min())
-                _max = float(df[column].max())
-
-                if (_min != _max) and pd.notna(_min) and pd.notna(_max):
-                    step = 0.01
-                    user_num_input = right.slider(
-                        f"Values for {column}:",
-                        min_value=round(_min - step, 2),
-                        max_value=round(_max + step, 2),
-                        value=(_min, _max),
-                        step=step,
-                    )
-                    df = df[df[column].between(*user_num_input)]
-
-            elif is_datetime64_any_dtype(df[column]):
-                user_date_input = right.date_input(
-                    f"Values for {column}:",
-                    value=(
-                        df[column].min(),
-                        df[column].max(),
-                    ),
-                )
-                if isinstance(user_date_input, Iterable) and len(user_date_input) == 2:
-                    user_date_input_datetime = tuple(map(pd.to_datetime, user_date_input))
-                    start_date, end_date = user_date_input_datetime
-                    df = df.loc[df[column].between(start_date, end_date)]
-
-            else:
-                selected_values = right.multiselect(
-                    f"Values for {column}:",
-                    sorted(df[column].unique()),
-                )
-
-                if selected_values:
-                    df = df[df[column].isin(selected_values)]
-
-    return df
-
-
-def setup_basic():
-    title = "🏆 LLM-Leaderboard"
-
-    st.set_page_config(
-        page_title=title,
-        page_icon="🏆",
-        layout="wide",
-    )
-    st.title(title)
-
-    st.markdown(
-        "A joint community effort to create one central leaderboard for LLMs."
-        f" Visit [swahili-llm-leaderboard]({GITHUB_URL}) to contribute.  \n"
-        'We refer to a model being "open" if it can be locally deployed and used for commercial purposes.'
-    )
-
-
-def setup_leaderboard(readme: str):
-    leaderboard_table = extract_markdown_table_from_multiline(readme, table_headline="## Leaderboard")
-    leaderboard_table = remove_markdown_links(leaderboard_table)
-    df_leaderboard = extract_table_and_format_from_markdown_text(leaderboard_table)
-    df_leaderboard["Open?"] = df_leaderboard["Open?"].map({"yes": 1, "no": 0}).astype(bool)
-
-    st.markdown("## Leaderboard")
-    modify = st.checkbox("Add filters")
-    clear_empty_entries = st.checkbox("Clear empty entries", value=True)
-
-    if modify:
-        df_leaderboard = filter_dataframe_by_row_and_columns(df_leaderboard, ignore_columns=NON_BENCHMARK_COLS)
-        df_leaderboard = filter_dataframe_by_column_values(df_leaderboard)
-
-    if clear_empty_entries:
-        df_leaderboard = df_leaderboard.dropna(axis=1, how="all")
-        benchmark_columns = [c for c in df_leaderboard.columns if df_leaderboard[c].dtype == float]
-        rows_wo_any_benchmark = df_leaderboard[benchmark_columns].isna().all(axis=1)
-        df_leaderboard = df_leaderboard[~rows_wo_any_benchmark]
-
-    st.dataframe(df_leaderboard)
-
-    st.download_button(
-        "Download current selection as .html",
-        df_leaderboard.to_html().encode("utf-8"),
-        "leaderboard.html",
-        "text/html",
-        key="download-html",
-    )
-
-    st.download_button(
-        "Download current selection as .csv",
-        df_leaderboard.to_csv().encode("utf-8"),
-        "leaderboard.csv",
-        "text/csv",
-        key="download-csv",
-    )
-
-
-def setup_benchmarks(readme: str):
-    benchmarks_table = extract_markdown_table_from_multiline(readme, table_headline="## Benchmarks")
-    df_benchmarks = extract_table_and_format_from_markdown_text(benchmarks_table)
-
-    st.markdown("## Covered Benchmarks")
-
-    selected_benchmark = st.selectbox("Select a benchmark to learn more:", df_benchmarks.index.unique())
-    df_selected = df_benchmarks.loc[selected_benchmark]
-    text = [
-        f"Name: {selected_benchmark}",
-    ]
-    for key in df_selected.keys():
-        text.append(f"{key}: {df_selected[key]} ")
-    st.markdown("  \n".join(text))
-
-
-def setup_sources():
-    st.markdown("## Sources")
-    st.markdown(
-        "The results of this leaderboard are collected from the individual papers and published results of the model "
-        "authors. If you are interested in the sources of each individual reported model value, please visit the "
-        f"[llm-leaderboard]({GITHUB_URL}) repository."
-    )
-    st.markdown(
-        """
-    Special thanks to the following pages:
-    - [MosaicML - Model benchmarks](https://www.mosaicml.com/blog/mpt-7b)
-    - [lmsys.org - Chatbot Arena benchmarks](https://lmsys.org/blog/2023-05-03-arena/)
-    - [Papers With Code](https://paperswithcode.com/)
-    - [Stanford HELM](https://crfm.stanford.edu/helm/latest/)
-    - [HF Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
-    """
-    )
-
-def setup_Contribution():
-    st.markdown("## How to Contribute")
-
-    markdown_content = """
-    - Model name (don't forget the links):
-      - Filling in missing entries
-      - Adding a new model as a new row to the leaderboard. Please keep the descending order.
-      - Adding a new benchmark as a new column in the leaderboard and adding the benchmark to the benchmarks table. Please keep the descending order.
-    - Code work:
-      - Improving the existing code
-      - Requesting and implementing new features
-    """
-    st.markdown(markdown_content)
+def display_leaderboard(df):
+    """Display the leaderboard."""
+    st.header("📊 Leaderboard")
     
-def setup_Sponsorship():
-    st.markdown("## Sponsorship")
-    st.markdown(
-        # "The results of this leaderboard are collected from the individual papers and published results of the model "
-        # "authors. If you are interested in the sources of each individual reported model value, please visit the "
-        # f"[llm-leaderboard]({GITHUB_URL}) repository."
-        "The benchmark is English-based, and we need support translating it into Swahili." 
-        "We welcome sponsorships to help advance this endeavor."
-        "Your sponsorship would facilitate this essential translation effort, bridging language barriers and making the benchmark "
-        "accessible to a broader audience. We're grateful for the dedication shown by our collaborators and aim to extend this impact "
-        "further with the support of sponsors committed to advancing language technologies."
-        "Any support please reach me: msamwelmollel@gmail.com"
-    )
-
-
-def setup_disclaimer():
-    st.markdown("## Disclaimer")
-    st.markdown(
-        "Above information may be wrong. If you want to use a published model for commercial use, please contact a "
-        "lawyer."
-    )
-
-
-def setup_footer():
-    st.markdown(
-        """
-        ---
-        Made with ❤️ by the awesome open-source community from all over 🌍.
-        """
-    )
-
+    # Add filters
+    columns_to_filter = [col for col in df.columns if col not in NON_BENCHMARK_COLS]
+    selected_columns = st.multiselect("Select benchmarks to display:", columns_to_filter, default=columns_to_filter)
+    
+    # Filter dataframe
+    df_display = df[NON_BENCHMARK_COLS + selected_columns]
+    
+    # Display dataframe
+    st.dataframe(df_display)
+    
+    # Download buttons
+    csv = df_display.to_csv(index=False)
+    st.download_button(label="Download as CSV", data=csv, file_name="leaderboard.csv", mime="text/csv")
+
+def display_evaluation():
+    """Display the evaluation section."""
+    st.header("🧪 Evaluation")
+    st.markdown("""
+    To evaluate a model on the Swahili Embeddings Text Benchmark, you can use the following Python script:
+    ```python
+    pip install mteb
+    pip install sentence-transformers
+    import mteb
+    from sentence_transformers import SentenceTransformer
+
+    models = ["sartifyllc/MultiLinguSwahili-bert-base-sw-cased-nli-matryoshka"]
+
+    for model_name in models:
+        truncate_dim = 768
+        language = "swa"
+        
+        device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
+        model = SentenceTransformer(model_name, device=device, trust_remote_code=True)
+        
+        tasks = [
+            mteb.get_task("AfriSentiClassification", languages=["swa"]),
+            mteb.get_task("AfriSentiLangClassification", languages=["swa"]),
+            mteb.get_task("MasakhaNEWSClassification", languages=["swa"]),
+            mteb.get_task("MassiveIntentClassification", languages=["swa"]),
+            mteb.get_task("MassiveScenarioClassification", languages=["swa"]),
+            mteb.get_task("SwahiliNewsClassification", languages=["swa"]),
+        ]
+        
+        evaluation = mteb.MTEB(tasks=tasks)
+        results = evaluation.run(model, output_folder=f"{model_name}")
+        
+        tasks = mteb.get_tasks(task_types=["PairClassification", "Reranking", "BitextMining", "Clustering", "Retrieval"], languages=["swa"])
+        
+        evaluation = mteb.MTEB(tasks=tasks)
+        results = evaluation.run(model, output_folder=f"{model_name}")
+    ```
+    """)
+
+def display_contribution():
+    """Display the contribution section."""
+    st.header("🤝 How to Contribute")
+    st.markdown("""
+    We welcome and appreciate all contributions! You can help by:
+
+    ### Table Work
+
+    - Filling in missing entries.
+    - New models are added as new rows to the leaderboard (maintaining descending order).
+    - Add new benchmarks as new columns in the leaderboard and include them in the benchmarks table (maintaining descending order).
+
+    ### Code Work
+
+    - Improving the existing code.
+    - Requesting and implementing new features.
+    """)
+
+def display_sponsorship():
+    """Display the sponsorship section."""
+    st.header("🤝 Sponsorship")
+    st.markdown("""
+    This benchmark is Swahili-based, and we need support translating and curating more tasks into Swahili. 
+    Sponsorships are welcome to help advance this endeavour. Your sponsorship will facilitate essential 
+    translation efforts, bridge language barriers, and make the benchmark accessible to a broader audience. 
+    We are grateful for the dedication shown by our collaborators and aim to extend this impact further 
+    with the support of sponsors committed to advancing language technologies.
+    """)
 
 def main():
-    setup_basic()
-
+    setup_page()
+    
+    # Read README content
     with open("README.md", "r") as f:
-        readme = f.read()
-  
-
-    setup_leaderboard(readme)
-    # setup_benchmarks(readme)
-    # setup_sources()
-    # setup_disclaimer()
-    # setup_footer()
-    setup_Contribution()
-    setup_Sponsorship()
-
+        readme_content = f.read()
+    
+    # Extract and process leaderboard table
+    leaderboard_table = extract_table_from_markdown(readme_content, "| Model Name")
+    df_leaderboard = markdown_table_to_df(leaderboard_table)
+    
+    display_leaderboard(df_leaderboard)
+    display_evaluation()
+    display_contribution()
+    display_sponsorship()
+    
+    st.markdown("---")
+    st.markdown("Thank you for being part of this effort to advance Swahili language technologies!")
 
 if __name__ == "__main__":
     main()