utils/smolagents_integration.py

import streamlit as st
import pandas as pd
import numpy as np

def process_with_smolagents(dataset, operation, custom_code=None):
    """
    Process dataset using SmolaAgents for various operations.
    
    Args:
        dataset: Pandas DataFrame to process
        operation: Type of processing operation
        custom_code: Custom code to execute (for custom processing)
        
    Returns:
        Processed pandas DataFrame
    """
    if dataset is None:
        raise ValueError("No dataset provided")
    
    # Create a copy to avoid modifying the original
    processed_df = dataset.copy()
    
    try:
        if operation == "Data Cleaning":
            processed_df = clean_dataset(processed_df)
        elif operation == "Feature Engineering":
            processed_df = engineer_features(processed_df)
        elif operation == "Data Transformation":
            processed_df = transform_dataset(processed_df)
        elif operation == "Custom Processing" and custom_code:
            # Execute custom code
            # Note: This is a security risk in a real application
            # Should be replaced with a safer approach
            local_vars = {"df": processed_df}
            exec(custom_code, {"pd": pd, "np": np}, local_vars)
            processed_df = local_vars["df"]
        else:
            raise ValueError(f"Unsupported operation: {operation}")
        
        return processed_df
    
    except Exception as e:
        st.error(f"Error during processing: {str(e)}")
        raise

def clean_dataset(df):
    """
    Clean the dataset by handling missing values, duplicates, and outliers.
    
    Args:
        df: Pandas DataFrame to clean
        
    Returns:
        Cleaned pandas DataFrame
    """
    # Create a copy to avoid modifying the original
    cleaned_df = df.copy()
    
    # Remove duplicate rows
    cleaned_df = cleaned_df.drop_duplicates()
    
    # Handle missing values
    for col in cleaned_df.columns:
        # For numeric columns
        if pd.api.types.is_numeric_dtype(cleaned_df[col]):
            # If more than 20% missing, leave as is
            if cleaned_df[col].isna().mean() > 0.2:
                continue
            
            # Otherwise impute with median
            cleaned_df[col] = cleaned_df[col].fillna(cleaned_df[col].median())
        
        # For categorical columns
        elif pd.api.types.is_object_dtype(cleaned_df[col]):
            # If more than 20% missing, leave as is
            if cleaned_df[col].isna().mean() > 0.2:
                continue
            
            # Otherwise impute with mode
            mode_value = cleaned_df[col].mode()[0] if not cleaned_df[col].mode().empty else "Unknown"
            cleaned_df[col] = cleaned_df[col].fillna(mode_value)
    
    # Handle outliers in numeric columns
    for col in cleaned_df.select_dtypes(include=[np.number]).columns:
        # Skip if too many missing values
        if cleaned_df[col].isna().mean() > 0.1:
            continue
        
        # Calculate IQR
        q1 = cleaned_df[col].quantile(0.25)
        q3 = cleaned_df[col].quantile(0.75)
        iqr = q3 - q1
        
        # Define bounds
        lower_bound = q1 - 1.5 * iqr
        upper_bound = q3 + 1.5 * iqr
        
        # Cap outliers instead of removing
        cleaned_df[col] = cleaned_df[col].clip(lower_bound, upper_bound)
    
    return cleaned_df

def engineer_features(df):
    """
    Perform basic feature engineering on the dataset.
    
    Args:
        df: Pandas DataFrame to process
        
    Returns:
        DataFrame with engineered features
    """
    # Create a copy to avoid modifying the original
    engineered_df = df.copy()
    
    # Get numeric columns
    numeric_cols = engineered_df.select_dtypes(include=[np.number]).columns
    
    # Skip if less than 2 numeric columns
    if len(numeric_cols) >= 2:
        # Create interaction features for pairs of numeric columns
        # Limit to first 5 columns to avoid feature explosion
        for i, col1 in enumerate(numeric_cols[:5]):
            for col2 in numeric_cols[i+1:5]:
                # Product interaction
                engineered_df[f"{col1}_{col2}_product"] = engineered_df[col1] * engineered_df[col2]
                
                # Ratio interaction (avoid division by zero)
                denominator = engineered_df[col2].replace(0, np.nan)
                engineered_df[f"{col1}_{col2}_ratio"] = engineered_df[col1] / denominator
    
    # Create binary features from categorical columns
    cat_cols = engineered_df.select_dtypes(include=['object', 'category']).columns
    for col in cat_cols:
        # Skip if too many unique values (>10)
        if engineered_df[col].nunique() > 10:
            continue
            
        # One-hot encode
        dummies = pd.get_dummies(engineered_df[col], prefix=col, drop_first=True)
        engineered_df = pd.concat([engineered_df, dummies], axis=1)
    
    # Create aggregated features
    if len(numeric_cols) >= 3:
        # Sum of all numeric features
        engineered_df['sum_numeric'] = engineered_df[numeric_cols].sum(axis=1)
        
        # Mean of all numeric features
        engineered_df['mean_numeric'] = engineered_df[numeric_cols].mean(axis=1)
        
        # Standard deviation of numeric features
        engineered_df['std_numeric'] = engineered_df[numeric_cols].std(axis=1)
    
    return engineered_df

def transform_dataset(df):
    """
    Perform data transformations on the dataset.
    
    Args:
        df: Pandas DataFrame to transform
        
    Returns:
        Transformed pandas DataFrame
    """
    from sklearn.preprocessing import StandardScaler, MinMaxScaler
    
    # Create a copy to avoid modifying the original
    transformed_df = df.copy()
    
    # Get numeric columns
    numeric_cols = transformed_df.select_dtypes(include=[np.number]).columns
    
    if len(numeric_cols) > 0:
        # Create scaled versions of numeric columns
        
        # Standard scaling (z-score)
        scaler = StandardScaler()
        scaled_data = scaler.fit_transform(transformed_df[numeric_cols])
        scaled_df = pd.DataFrame(
            scaled_data, 
            columns=[f"{col}_scaled" for col in numeric_cols],
            index=transformed_df.index
        )
        
        # Min-max scaling (0-1 range)
        minmax_scaler = MinMaxScaler()
        minmax_data = minmax_scaler.fit_transform(transformed_df[numeric_cols])
        minmax_df = pd.DataFrame(
            minmax_data,
            columns=[f"{col}_normalized" for col in numeric_cols],
            index=transformed_df.index
        )
        
        # Log transform (for positive columns only)
        log_cols = []
        for col in numeric_cols:
            if (transformed_df[col] > 0).all():
                transformed_df[f"{col}_log"] = np.log(transformed_df[col])
                log_cols.append(f"{col}_log")
        
        # Combine all transformations
        transformed_df = pd.concat([transformed_df, scaled_df, minmax_df], axis=1)
    
    # One-hot encode categorical columns
    cat_cols = transformed_df.select_dtypes(include=['object', 'category']).columns
    if len(cat_cols) > 0:
        # One-hot encode all categorical columns
        transformed_df = pd.get_dummies(transformed_df, columns=cat_cols, drop_first=False)
    
    return transformed_df