更新了结合Ploty动态交互的功能，以及展示统计值的功能

README.md

@@ -13,6 +13,7 @@ pinned: false
 
 Access any slice of data of any dataset on the [Hugging Face Dataset Hub](https://huggingface.co/datasets)
 
+This project is modified based on the project https://huggingface.co/spaces/lhoestq/datasets-explorer
 Run:
 
 ```python

app.py

@@ -1,24 +1,20 @@
-import base64
 import copy
-from datetime import datetime, timedelta
-from io import BytesIO
-import random
+import os
+from functools import lru_cache, partial
+
 import gradio as gr
-from functools import lru_cache
-from hffs.fs import HfFileSystem
-from typing import List, Tuple, Callable
-from matplotlib import pyplot as plt
-import pandas as pd
 import numpy as np
+import pandas as pd
 import pyarrow as pa
 import pyarrow.parquet as pq
-from functools import partial
 from tqdm.contrib.concurrent import thread_map
-from datasets import Features, Image, Audio, Sequence
 from fastapi import FastAPI, Response
 import uvicorn
-import os
-from gradio_datetimerange import DateTimeRange
+from hffs.fs import HfFileSystem
+from datasets import Features, Image, Audio, Sequence
+from typing import List, Tuple, Callable
+
+from utils import ndarray_to_base64, clean_up_df, create_statistic, create_plot
 
 class AppError(RuntimeError):
     pass
@@ -30,46 +26,7 @@ MAX_CACHED_BLOBS = PAGE_SIZE * 10
 TIME_PLOTS_NUM = 5
 _blobs_cache = {}
 
-#####################################################
-# Utils
-#####################################################
-def ndarray_to_base64(ndarray):
-    """
-    将一维np.ndarray绘图并转换为Base64编码。
-    """
-    # 创建绘图
-    plt.figure(figsize=(8, 4))
-    plt.plot(ndarray)
-    plt.title("Vector Plot")
-    plt.xlabel("Index")
-    plt.ylabel("Value")
-    plt.tight_layout()
-    
-    # 保存图像到内存字节流
-    buffer = BytesIO()
-    plt.savefig(buffer, format="png")
-    plt.close()
-    buffer.seek(0)
-    
-    # 转换为Base64字符串
-    base64_str = base64.b64encode(buffer.getvalue()).decode('utf-8')
-    return f"data:image/png;base64,{base64_str}"
-
-def flatten_ndarray_column(df, column_name):
-    def flatten_ndarray(ndarray):
-        if isinstance(ndarray, np.ndarray) and ndarray.dtype == 'O':
-            return np.concatenate([flatten_ndarray(subarray) for subarray in ndarray])
-        elif isinstance(ndarray, np.ndarray) and ndarray.ndim == 1:
-            return np.expand_dims(ndarray, axis=0)
-        return ndarray
-
-    flattened_data = df[column_name].apply(flatten_ndarray)
-    max_length = max(flattened_data.apply(len))
-
-    for i in range(max_length):
-        df[f'{column_name}_{i}'] = flattened_data.apply(lambda x: x[i] if i < len(x) else np.nan)
 
-    return df
 #####################################################
 # Define routes for image and audio files
 #####################################################
@@ -239,7 +196,7 @@ def get_page(dataset: str, config: str, split: str, page: str) -> Tuple[str, int
         info = "" if not unsupported_columns else f"Some columns are not supported yet: {unsupported_columns}"
         return df.reset_index().to_markdown(index=False), max_page, info
     else:
-        # 其他的处理逻辑
+        # 对Salesforce/lotsa_data数据集进行特殊处理
         info = "" if not unsupported_columns else f"Some columns are not supported yet: {unsupported_columns}"
         return df, max_page, info
 
@@ -250,6 +207,7 @@ def get_page(dataset: str, config: str, split: str, page: str) -> Tuple[str, int
 
 
 with gr.Blocks() as demo:
+    # 初始化组件
     gr.Markdown("# 📖 Datasets Explorer\n\nAccess any slice of data of any dataset on the [Hugging Face Dataset Hub](https://huggingface.co/datasets)")
     gr.Markdown("This is the dataset viewer from parquet export demo before the feature was added on the Hugging Face website.")
     cp_dataset = gr.Textbox("Salesforce/lotsa_data", label="Pick a dataset", placeholder="competitions/aiornot")
@@ -261,32 +219,16 @@ with gr.Blocks() as demo:
     cp_info = gr.Markdown("", visible=False)
     cp_result = gr.Markdown("", visible=False)
 
-    now = datetime.now()
-    df = pd.DataFrame({
-        'time': [now - timedelta(minutes=5*i) for i in range(25)] + [now],
-        'price': np.random.randint(100, 1000, 26),
-        'origin': [random.choice(["DFW", "DAL", "HOU"]) for _ in range(26)],
-        'destination': [random.choice(["JFK", "LGA", "EWR"]) for _ in range(26)],
-    })
-
+    # 初始化Salesforce/lotsa_data数据集展示使用的组件
     componets = []
     for _ in range(TIME_PLOTS_NUM):
         with gr.Row():
-            textbox = gr.Textbox("名称或说明")
-            with gr.Column():
-                daterange = DateTimeRange(["now - 24h", "now"])
-                plot1 = gr.LinePlot(df, x="time", y="price", color="origin")
-                # plot2 = gr.LinePlot(df, x="time", y="price", color="origin")
-                daterange.bind([plot1, 
-                                # plot2,
-                                ])
-                comp = {
-                    "textbox" : textbox,
-                    "daterange" : daterange,
-                    "plot1" : plot1,
-                    # "plot2" : plot2,
-                }
-                componets.append(comp)
+            with gr.Column(scale=2):
+                textbox = gr.Textbox("名称或说明")
+                statistics_textbox = gr.DataFrame()
+            with gr.Column(scale=3):
+                plot = gr.Plot()
+            componets.append({"textbox": textbox, "statistics_textbox": statistics_textbox, "plot": plot})
 
     with gr.Row():
         cp_page = gr.Textbox("1", label="Page", placeholder="1", visible=False)
@@ -306,24 +248,19 @@ with gr.Blocks() as demo:
                 markdown_result, max_page, info = get_page(dataset, config, split, page)
                 ret[cp_result] = gr.update(visible=True, value=markdown_result)
             else:
+                # 对Salesforce/lotsa_data数据集进行特殊处理
                 df, max_page, info = get_page(dataset, config, split, page)
-                print(df.columns)
-                # TODO:target为一维数组时len(row['target'][0])会直接报错
-                df['timestamp'] = df.apply(lambda row: pd.date_range(start=row['start'], periods=len(row['target'][0]), freq=row['freq']).to_pydatetime().tolist(), axis=1)
-                df = flatten_ndarray_column(df, 'target')
-                # 删除原始的start和freq列
-                df.drop(columns=['start', 'freq', 'target'], inplace=True)
-                if 'past_feat_dynamic_real' in df.columns:
-                    df.drop(columns=['past_feat_dynamic_real'], inplace=True)
-                info = f"({info})" if info else ""
+                df = clean_up_df(df)
                 for i, rows in df.iterrows():
                     index = rows['item_id']
+                    # 将单行的DataFrame展开为新的DataFrame
                     df_without_index = rows.drop('item_id').to_frame().T
                     df_expanded = df_without_index.apply(pd.Series.explode).reset_index(drop=True).fillna(0)
+                    df_statistics = create_statistic(df_expanded)
                     ret.update({
                         componets[i]["textbox"]: gr.update(value=f"item_id: {index}"),
-                        componets[i]["daterange"]: gr.update(value=[df_without_index['timestamp'][i][0], df_without_index['timestamp'][i][-1]]),
-                        componets[i]["plot1"]: gr.update(value=df_expanded, x="timestamp", y="target_0"),
+                        componets[i]["statistics_textbox"]: gr.update(value=df_statistics),
+                        componets[i]["plot"]: gr.update(value=create_plot(df_expanded))
                     })
             return {
                 **ret,
@@ -381,34 +318,10 @@ with gr.Blocks() as demo:
             }
         except AppError as err:
             return show_error(str(err))
-    
-    """
-    动态生成组件时使用gr.LinePlot会有bug，直接卡死在show_dataset部分
-    """
-    # @gr.render(triggers=[cp_go.click])
-    # def create_test():
-    #     now = datetime.now()
-    #     df = pd.DataFrame({
-    #     'time': [now - timedelta(minutes=5*i) for i in range(25)],
-    #     'price': np.random.randint(100, 1000, 25),
-    #     'origin': [random.choice(["DFW", "DAL", "HOU"]) for _ in range(25)],
-    #     'destination': [random.choice(["JFK", "LGA", "EWR"]) for _ in range(25)],
-    #     })
-    #     # componets = []
-    #     # daterange = DateTimeRange(["now - 24h", "now"])
-    #     plot1 = gr.LinePlot(df, x="time", y="price")
-    #     plot2 = gr.LinePlot(df, x="time", y="price", color="origin")
-    #     # # daterange.bind([plot1, plot2])
-    #     # componets.append(plot1)
-    #     # componets.append(plot2)
-    #     # componets.append(daterange)
-    #     # test = gr.Textbox(label="input")
-    #     # componets.append(test)
-    #     # return componets
 
     all_outputs = [cp_config, cp_split, cp_page, cp_goto_page, cp_goto_next_page, cp_result, cp_info, cp_error]
-    for comp in componets:
-        all_outputs += list(comp.values())
+    for componet in componets:
+        all_outputs += list(componet.values())
     cp_go.click(show_dataset, inputs=[cp_dataset], outputs=all_outputs)
     cp_config.change(show_dataset_at_config, inputs=[cp_dataset, cp_config], outputs=all_outputs)
     cp_split.change(show_dataset_at_config_and_split, inputs=[cp_dataset, cp_config, cp_split], outputs=all_outputs)

utils.py

@@ -0,0 +1,119 @@
+#####################################################
+# Utils
+#####################################################
+# 本文件包含了一些用于数据处理和绘图的实用函数。
+
+import base64
+from io import BytesIO
+from matplotlib import pyplot as plt
+import pandas as pd
+import plotly.graph_objects as go
+import numpy as np
+
+
+def ndarray_to_base64(ndarray):
+    """
+    将一维np.ndarray绘图并转换为Base64编码。
+    """
+    # 创建绘图
+    plt.figure(figsize=(8, 4))
+    plt.plot(ndarray)
+    plt.title("Vector Plot")
+    plt.xlabel("Index")
+    plt.ylabel("Value")
+    plt.tight_layout()
+    
+    # 保存图像到内存字节流
+    buffer = BytesIO()
+    plt.savefig(buffer, format="png")
+    plt.close()
+    buffer.seek(0)
+    
+    # 转换为Base64字符串
+    base64_str = base64.b64encode(buffer.getvalue()).decode('utf-8')
+    return f"data:image/png;base64,{base64_str}"
+
+def flatten_ndarray_column(df, column_name):
+    """
+    将嵌套的np.ndarray列展平为多列。
+    """
+    def flatten_ndarray(ndarray):
+        if isinstance(ndarray, np.ndarray) and ndarray.dtype == 'O':
+            return np.concatenate([flatten_ndarray(subarray) for subarray in ndarray])
+        elif isinstance(ndarray, np.ndarray) and ndarray.ndim == 1:
+            return np.expand_dims(ndarray, axis=0)
+        return ndarray
+
+    flattened_data = df[column_name].apply(flatten_ndarray)
+    max_length = max(flattened_data.apply(len))
+
+    for i in range(max_length):
+        df[f'{column_name}_{i}'] = flattened_data.apply(lambda x: x[i] if i < len(x) else np.nan)
+
+    return df
+
+def create_plot(df):
+    """
+    创建一个包含所有列的线图。
+    """
+    fig = go.Figure()
+    for i, column in enumerate(df.columns[1:]):
+        fig.add_trace(go.Scatter(
+            x=df[df.columns[0]], 
+            y=df[column], 
+            mode='lines', 
+            name=column,
+            visible=True if i == 0 else 'legendonly'
+        ))
+    
+    # 配置图例
+    fig.update_layout(
+        legend=dict(
+            title="Variables",
+            orientation="h",
+            yanchor="top",
+            y=-0.2,
+            xanchor="center",
+            x=0.5
+        ),
+        xaxis_title='Time',
+        yaxis_title='Values'
+    )
+    return fig
+
+def create_statistic(df):
+    """
+    计算数据集的统计信息。
+    """
+    df_values = df.iloc[:, 1:]
+    # 计算统计值
+    mean_values = df_values.mean()
+    std_values = df_values.std()
+    max_values = df_values.max()
+    min_values = df_values.min()
+
+    # 将这些统计信息合并成一个新的DataFrame
+    stats_df = pd.DataFrame({
+        'Variables': df_values.columns,
+        'mean': mean_values.values,
+        'std': std_values.values,
+        'max': max_values.values,
+        'min': min_values.values
+    })
+    return stats_df
+
+def clean_up_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    清理数据集，将嵌套的np.ndarray列展平为多列。
+    """
+    df['timestamp'] = df.apply(lambda row: pd.date_range(
+        start=row['start'], 
+        periods=len(row['target'][0]) if isinstance(row['target'][0], np.ndarray) else len(row['target']), 
+        freq=row['freq']
+    ).to_pydatetime().tolist(), axis=1)
+    df = flatten_ndarray_column(df, 'target')
+    # 删除原始的start和freq列
+    df.drop(columns=['start', 'freq', 'target'], inplace=True)
+    if 'past_feat_dynamic_real' in df.columns:
+        df.drop(columns=['past_feat_dynamic_real'], inplace=True)
+    return df