pages/clustering.py

from collections import defaultdict
import streamlit as st
from utils import load_and_preprocess_data
import pandas as pd
import numpy as np
import altair as alt
from sklearn.mixture import GaussianMixture
import plotly.express as px
import itertools
from typing import Dict, List

SIDEBAR_DESCRIPTION = """
# Client clustering

To cluster a client, we adopt the RFM metrics. They stand for:

- R = recency, that is the number of days since the last purchase 
    in the store
- F = frequency, that is the number of times a customer has ordered something
- M = monetary value, that is how much a customer has spent buying 
    from your business.

Given these 3 metrics, we can cluster the customers and find a suitable
"definition" based on the clusters they belong to. Since the dataset
we're using right now as about 5000 distinct customers, we identify
3 clusters for each metric.

## How we compute the clusters

We resort to a simple KMeans algorithm. It tries to find the clusters
based on the distance between points. In particular, near points tend to be associated
with the same cluster, while further points should belong to different clusters.
""".lstrip()

FREQUENCY_CLUSTERS_EXPLAIN = """
The **frequency** denotes how frequently a customer has ordered.

There 3 available clusters for this metric:

- cluster 0: denotes a customer that purchases one or few times (range [{}, {}])
- cluster 1: these customer have a discrete amount of orders (range [{}, {}])
- cluster 2: these customer purchases lots of times (range [{}, {}])

-------
""".lstrip()

RECENCY_CLUSTERS_EXPLAIN = """
The **recency** refers to how recently a customer has bought; 

There 3 available clusters for this metric:

- cluster 0: the last order of these client is long time ago (range [{}, {}])
- cluster 1: these are clients that purchases something not very recently (range [{}, {}])
- cluster 2: the last order of these client is a few days/weeks ago (range [{}, {}])

-------
""".lstrip()

MONETARY_CLUSTERS_EXPLAIN = """
The **revenue** refers to how much a customer has spent buying 
from your business.

There 3 available clusters for this metric:

- cluster 0: these clients spent little money (range [{}, {}])
- cluster 1: these clients spent a considerable amount of money (range [{}, {}])
- cluster 2: these clients spent lots of money (range [{}, {}])

-------
""".lstrip()

EXPLANATION_DICT = {
    "Frequency_cluster": FREQUENCY_CLUSTERS_EXPLAIN,
    "Recency_cluster": RECENCY_CLUSTERS_EXPLAIN,
    "Revenue_cluster": MONETARY_CLUSTERS_EXPLAIN,
}


def create_features(df: pd.DataFrame):
    """Creates a new dataframe with the RFM features for each client."""
    # Compute frequency, the number of distinct time a user purchased.
    client_features = df.groupby("CustomerID")["InvoiceDate"].nunique().reset_index()
    client_features.columns = ["CustomerID", "Frequency"]

    # Add monetary value, the total revenue for  each single user.
    client_takings = df.groupby("CustomerID")["Price"].sum()
    client_features["Revenue"] = client_takings.values

    # Add recency, i.e. the days since the last purchase in the store.
    max_date = df.groupby("CustomerID")["InvoiceDate"].max().reset_index()
    max_date.columns = ["CustomerID", "LastPurchaseDate"]

    client_features["Recency"] = (
        max_date["LastPurchaseDate"].max() - max_date["LastPurchaseDate"]
    ).dt.days

    return client_features


@st.cache
def cluster_clients(df: pd.DataFrame):
    """Computes the RFM features and clusters for each user based on the RFM metrics."""

    df_rfm = create_features(df)

    for to_cluster, order in zip(
        ["Revenue", "Frequency", "Recency"], ["ascending", "ascending", "descending"]
    ):
        kmeans = GaussianMixture(n_components=3, random_state=42)
        labels = kmeans.fit_predict(df_rfm[[to_cluster]])
        df_rfm[f"{to_cluster}_cluster"] = _order_cluster(kmeans, labels, order)

    return df_rfm


def _order_cluster(cluster_model: GaussianMixture, clusters, order="ascending"):
    """Orders the cluster by order."""
    centroids = cluster_model.means_.sum(axis=1)

    if order.lower() == "descending":
        centroids *= -1

    ascending_order = np.argsort(centroids)
    lookup_table = np.zeros_like(ascending_order)
    # Cluster will start from 1
    lookup_table[ascending_order] = np.arange(cluster_model.n_components) + 1
    return lookup_table[clusters]


def show_purhcase_history(user: int, df: pd.DataFrame):
    user_purchases = df.loc[df.CustomerID == user, ["Price", "InvoiceDate"]]
    expenses = user_purchases.groupby(user_purchases.InvoiceDate).sum()
    expenses.columns = ["Expenses"]
    expenses = expenses.reset_index()

    c = (
        alt.Chart(expenses)
        .mark_line(point=True)
        .encode(
            x=alt.X("InvoiceDate", timeUnit="yearmonthdate", title="Date"),
            y="Expenses",
        )
        .properties(title="User expenses")
    )

    st.altair_chart(c, use_container_width=True)


def show_user_info(user: int, df_rfm: pd.DataFrame):
    """Prints some information about the user.

    The main information are the total expenses, how
    many times he purchases in the store, and the clusters
    he belongs to.
    """

    user_row = df_rfm[df_rfm["CustomerID"] == user]
    if len(user_row) == 0:
        st.write(f"No user with id {user}")

    output = []

    output.append(f"The user purchased **{user_row['Frequency'].squeeze()} times**.\n")
    output.append(
        f"She/he spent **{user_row['Revenue'].squeeze()} dollars** in total.\n"
    )
    output.append(
        f"The last time she/he bought something was **{user_row['Recency'].squeeze()} days ago**.\n"
    )
    output.append(f"She/he belongs to the clusters: ")
    for cluster in [column for column in user_row.columns if "_cluster" in column]:
        output.append(f"- {cluster} = {user_row[cluster].squeeze()}")

    st.write("\n".join(output))

    return (
        user_row["Recency_cluster"].squeeze(),
        user_row["Frequency_cluster"].squeeze(),
        user_row["Revenue_cluster"].squeeze(),
    )


def explain_cluster(cluster_info):
    """Displays a popup menu explinging the meanining of the clusters."""

    with st.expander("Show information about the clusters"):
        st.write(
            "**Note**: these values are valid for these dataset."
            "Different dataset will have different number of clusters"
            " and values"
        )
        for cluster, info in cluster_info.items():
            st.write(EXPLANATION_DICT[cluster].format(*info))


def categorize_user(recency_cluster, frequency_cluster, monetary_cluster):
    """Describe the user with few words based on the cluster he belongs to."""

    score = f"{recency_cluster}{frequency_cluster}{monetary_cluster}"

    # @fixme: find a better approeach. These elif chains don't scale at all.

    description = ""

    if score == "111":
        description = "Tourist"
    elif score.startswith("2"):
        description = "Losing interest"
    elif score == "133":
        description = "Former lover"
    elif score == "123":
        description = "Former passionate client"
    elif score == "113":
        description = "Spent a lot, but never come back"
    elif score.startswith("1"):
        description = "About to dump"
    elif score == "313":
        description = "Potential lover"
    elif score == "312":
        description = "Interesting new client"
    elif score == "311":
        description = "New customer"
    elif score == "333":
        description = "Gold client"
    elif score == "322":
        description = "Lovers"
    else:
        description = "Average client"

    st.write(f"The customer can be described as: **{description}**")


def plot_rfm_distribution(df_rfm: pd.DataFrame, cluster_info: Dict[str, List[int]]):
    """Plots 3 histograms for the RFM metrics."""

    for x in ("Revenue", "Frequency", "Recency"):
        fig = px.histogram(df_rfm, x=x, log_y=True, title=f"{x} metric")
        # Get the max value in the cluster info. The cluster info is a list of min - max
        # values per cluster.
        values = cluster_info[f"{x}_cluster"]
        for n_cluster, i in enumerate(range(1, len(values), 2)):
            fig.add_vline(
                x=values[i],
                annotation_text=f"End of cluster {n_cluster+1}",
                line_dash="dot",
                annotation=dict(textangle=90, font_color="red"),
            )
        st.plotly_chart(fig, use_container_width=True)


def display_dataframe_heatmap(df_rfm: pd.DataFrame):
    """Displays an heatmap of how many clients lay in the clusters.

    This method uses some black magic coming from the dataframe
    styling guide.
    """

    # Create a dataframe with the count of clients for each group
    # of cluster.

    count = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
            "CustomerID"
        ]
        .count()
        .reset_index()
    )
    count = count.rename(columns={"CustomerID": "Count"})

    # Remove duplicates
    count = count.drop_duplicates(
        ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]
    )

    # Use the count column as values, then index with the clusters.
    count = count.pivot(
        index=["Revenue_cluster", "Frequency_cluster"],
        columns="Recency_cluster",
        values="Count",
    )

    # Style manipulation
    cell_hover = {
        "selector": "td",
        "props": "font-size:1.5em",
    }
    index_names = {
        "selector": ".index_name",
        "props": "font-style: italic; color: Black; font-weight:normal;font-size:1.5em;",
    }
    headers = {
        "selector": "th:not(.index_name)",
        "props": "background-color: White; color: black; font-size:1.5em",
    }

    # Finally, display
    # We cannot directly print the dataframe since the streamlit
    # functin remove the multiindex. Thus, we extract the html representation
    # and then display it.
    st.markdown("## Heatmap: how the client are distributed between clusters")
    st.write(
        count.style.format(thousands=" ", precision=0, na_rep="Missing")
        .set_table_styles([cell_hover, index_names, headers])
        .background_gradient(cmap="coolwarm")
        .to_html(),
        unsafe_allow_html=True,
    )


def main():
    st.sidebar.markdown(SIDEBAR_DESCRIPTION)

    df, _, _ = load_and_preprocess_data()
    df_rfm = cluster_clients(df)

    st.markdown(
        "# Dataset "
        "\nThis is the processed dataset with information about the clients, such as"
        " the RFM values and the clusters they belong to."
        )
    st.dataframe(df_rfm)

    cluster_info_dict = defaultdict(list)

    with st.expander("Show more details about the clusters"):
        for cluster in [column for column in df_rfm.columns if "_cluster" in column]:
            st.write(cluster)
            cluster_info = (
                df_rfm.groupby(cluster)[cluster.split("_")[0]]
                .describe()
                .reset_index(names="Cluster")
            )
            min_cluster = cluster_info["min"].astype(int)
            max_cluster = cluster_info["max"].astype(int)
            min_max_interlieved = list(itertools.chain(*zip(min_cluster, max_cluster)))
            cluster_info_dict[cluster].extend(min_max_interlieved)
            st.dataframe(cluster_info)

    st.markdown("## RFM metric distribution")

    plot_rfm_distribution(df_rfm, cluster_info_dict)

    display_dataframe_heatmap(df_rfm)

    st.markdown("## Interactive exploration")

    filter_by_cluster = st.checkbox(
        "Filter client: only one client per cluster type",
        value=True,
    )

    client_to_select = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])["CustomerID"].first().values 
        if filter_by_cluster 
        else df["CustomerID"].unique() 
    )

    # Let the user select the user to investigate
    user = st.selectbox(
        "Select a customer to show more information about him.",
        client_to_select,
    )

    show_purhcase_history(user, df)

    recency, frequency, revenue = show_user_info(user, df_rfm)

    categorize_user(recency, frequency, revenue)

    explain_cluster(cluster_info_dict)


main()