model.py

"""
track_id: The Spotify ID for the track
artists: The artists' names who performed the track. If there is more than one artist, they are separated by a ;
album_name: The album name in which the track appears
track_name: Name of the track
popularity: The popularity of a track is a value between 0 and 100, with 100 being the most popular. The popularity is calculated by algorithm and is based, in the most part, on the total number of plays the track has had and how recent those plays are. Generally speaking, songs that are being played a lot now will have a higher popularity than songs that were played a lot in the past. Duplicate tracks (e.g. the same track from a single and an album) are rated independently. Artist and album popularity is derived mathematically from track popularity.
duration_ms: The track length in milliseconds
explicit: Whether the track has explicit lyrics (true = yes it does; false = no it does not OR unknown)
danceability: Danceability describes how suitable a track is for dancing based on a combination of musical elements including tempo, rhythm stability, beat strength, and overall regularity. A value of 0.0 is least danceable and 1.0 is most danceable
energy: Energy is a measure from 0.0 to 1.0 and represents a perceptual measure of intensity and activity. Typically, energetic tracks feel fast, loud, and noisy. For example, death metal has high energy, while a Bach prelude scores low on the scale
key: The key the track is in. Integers map to pitches using standard Pitch Class notation. E.g. 0 = C, 1 = C♯/D♭, 2 = D, and so on. If no key was detected, the value is -1
loudness: The overall loudness of a track in decibels (dB)
mode: Mode indicates the modality (major or minor) of a track, the type of scale from which its melodic content is derived. Major is represented by 1 and minor is 0
speechiness: Speechiness detects the presence of spoken words in a track. The more exclusively speech-like the recording (e.g. talk show, audio book, poetry), the closer to 1.0 the attribute value. Values above 0.66 describe tracks that are probably made entirely of spoken words. Values between 0.33 and 0.66 describe tracks that may contain both music and speech, either in sections or layered, including such cases as rap music. Values below 0.33 most likely represent music and other non-speech-like tracks
acousticness: A confidence measure from 0.0 to 1.0 of whether the track is acoustic. 1.0 represents high confidence the track is acoustic
instrumentalness: Predicts whether a track contains no vocals. "Ooh" and "aah" sounds are treated as instrumental in this context. Rap or spoken word tracks are clearly "vocal". The closer the instrumentalness value is to 1.0, the greater likelihood the track contains no vocal content
liveness: Detects the presence of an audience in the recording. Higher liveness values represent an increased probability that the track was performed live. A value above 0.8 provides strong likelihood that the track is live
valence: A measure from 0.0 to 1.0 describing the musical positiveness conveyed by a track. Tracks with high valence sound more positive (e.g. happy, cheerful, euphoric), while tracks with low valence sound more negative (e.g. sad, depressed, angry)
tempo: The overall estimated tempo of a track in beats per minute (BPM). In musical terminology, tempo is the speed or pace of a given piece and derives directly from the average beat duration
time_signature: An estimated time signature. The time signature (meter) is a notational convention to specify how many beats are in each bar (or measure). The time signature ranges from 3 to 7 indicating time signatures of 3/4, to 7/4.
track_genre: The genre in which the track belongs
"""
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import cross_validate, GridSearchCV
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error
import spotipy
from spotipy.oauth2 import SpotifyOAuth
import random
from datetime import datetime

pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)


df_ = pd.read_csv(r"D:\Users\hhhjk\pythonProject\spotify_danceability\dataset.csv")

df = df_.copy()

#######################################################################
# 1. Exploratory Data Analysis
#######################################################################
df.info()

df = df[df.track_genre != 'kids']

df = df[df.track_genre != 'children']

df = df[df.track_genre != 'study']

df.track_genre.unique()

outcome = 'danceability'

df['time_signature'].unique()

df.drop("explicit", axis=1, inplace=True)

df.drop("Unnamed: 0", axis=1, inplace=True)

df['time_signature'] = df['time_signature'].replace({0: 6, 1: 7})

df.drop(65900, axis=0, inplace=True)


duplicated_rows = df[df.duplicated(subset=['track_id'])]
df = df.drop(duplicated_rows.index)

duplicated_rows = df[df.duplicated(subset=['track_id', 'artists', "album_name", "track_name"])]
df = df.drop(duplicated_rows.index)

duplicated_rows = df[df.duplicated(subset=['track_id', "track_name"])]
df = df.drop(duplicated_rows.index)

duplicated_rows = df[df.duplicated(subset=['track_id', 'artists', "album_name"])]
df = df.drop(duplicated_rows.index)

duplicated_rows = df[df.duplicated(subset=['popularity', 'duration_ms', "danceability", "energy", "key", "loudness",
                                           "mode", "speechiness", "acousticness", "instrumentalness", "liveness",
                                           "valence", "tempo", "time_signature"])]

df = df.drop(duplicated_rows.index)


def check_df(dataframe, head=5):
    print("#################### Shape ######################")
    print(dataframe.shape)
    print("#################### Types ######################")
    print(dataframe.dtypes)
    print("#################### Head #######################")
    print(dataframe.head(head))
    print("#################### Tail #######################")
    print(dataframe.tail(head))
    print("#################### NA #########################")
    print(dataframe.isnull().sum())
    print("#################### Quantiles ##################")
    print(dataframe.quantile([0, 0.05, 0.50, 0.95, 0.99, 1], numeric_only=True).T)


check_df(df)


def grab_col_names(dataframe, cat_th=13, car_th=20):
    """
    Veri setindeki kategorik, numerik ve kategorik fakat kardinal değişkenlerin isimlerini verir.

    Parameters
    ----------
    dataframe: dataframe
        değişken isimleri alınmak istenen dataframe'dir.
    cat_th: int, float
        numerik fakat kategorik olan değişkenler için sınıf eşik değeri
    car_th: int, float
        kategorik fakat kardinal değişkenler için sınıf eşik değeri

    Returns
    -------
    cat_cols: list
        Kategorik değişken listesi
    num_cols: list
        Numerik değişken listesi
    cat_but_car: list
        Kategorik görünümlü kardinal değişken listesi

    Notes
    ------
    cat_cols + num_cols + cat_but_car = toplam değişken sayısı
    num_but_cat cat_cols'un içerisinde.

    """
    # cat_cols, cat_but_car
    cat_cols = [col for col in df.columns if str(df[col].dtypes) in ["category", "object", "bool"]]

    num_but_cat = [col for col in df.columns if df[col].nunique() < cat_th and df[col].dtypes in ["int", "float"]]

    cat_but_car = [col for col in df.columns if
                   df[col].nunique() > car_th and str(df[col].dtypes) in ["category", "object"]]

    cat_cols = cat_cols + num_but_cat
    cat_cols = [col for col in cat_cols if col not in cat_but_car]

    num_cols = [col for col in df.columns if df[col].dtypes in ["int", "float"]]
    num_cols = [col for col in num_cols if col not in cat_cols]

    print(f"Observations: {dataframe.shape[0]}")
    print(f"Variables: {dataframe.shape[1]}")
    print(f'cat_cols: {len(cat_cols)}')
    print(f'num_cols: {len(num_cols)}')
    print(f'cat_but_car: {len(cat_but_car)}')
    print(f'num_but_cat: {len(num_but_cat)}')

    return cat_cols, num_cols, cat_but_car


cat_cols, num_cols, cat_but_car = grab_col_names(df)

num_cols.remove(outcome)



def cat_summary(dataframe, col_name, plot=False):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")
    if plot:
        sns.countplot(x=dataframe[col_name], data=dataframe)
        plt.show()


for col in cat_cols:
    cat_summary(df, col, plot=False)


def num_summary(dataframe, numerical_col, plot=False):
    quantiles = [0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99]
    print(dataframe[numerical_col].describe(quantiles).T)

    if plot:
        dataframe[numerical_col].hist(bins=20)
        plt.xlabel(numerical_col)
        plt.title(numerical_col)
        plt.show(block=True)


for col in num_cols:
    num_summary(df, col)


def target_summary_with_cat(dataframe, target, categorical_col):
    print(pd.DataFrame({"TARGET_MEAN": dataframe.groupby(categorical_col)[target].mean()}), end="\n\n\n")


for col in cat_cols:
    target_summary_with_cat(df, outcome, col)


def correlation_matrix(df, cols):
    fig = plt.gcf()
    fig.set_size_inches(10, 8)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    fig = sns.heatmap(df[cols].corr(), annot=True, linewidths=0.5, annot_kws={"size": 12}, linecolor="w", cmap="RdBu")
    plt.show(block=True)


correlation_matrix(df, num_cols)

#######################################################################
# 2. Data Preprocessing & Feature Engineering
#######################################################################


def outlier_thresholds(dataframe, col_name, q1=0.05, q3=0.95):
    quartile1 = dataframe[col_name].quantile(q1)
    quartile3 = dataframe[col_name].quantile(q3)
    interquantile_range = quartile3 - quartile1
    up_limit = quartile3 + 1.5 * interquantile_range
    low_limit = quartile1 - 1.5 * interquantile_range
    return low_limit, up_limit


def replace_with_thresholds(dataframe, variable):
    low_limit, up_limit = outlier_thresholds(dataframe, variable)
    dataframe.loc[(dataframe[variable] < low_limit), variable] = low_limit
    dataframe.loc[(dataframe[variable] > up_limit), variable] = up_limit


def check_outlier(dataframe, col_name):
    low_limit, up_limit = outlier_thresholds(dataframe, col_name)
    if dataframe[(dataframe[col_name] > up_limit) | (dataframe[col_name] < low_limit)].any(axis=None):
        return True
    else:
        return False



####################################################################################################################



for col in num_cols:
    print(col, ":", check_outlier(df, col))


for col in num_cols:
    replace_with_thresholds(df, col)


for col in num_cols:
    print(col, ":", check_outlier(df, col))

####################################################################################################################
#TASK - Encoding & Features


df.style.set_properties(**{'text-align': 'center'})
df.head()

df = pd.get_dummies(df, columns=["key"], drop_first=True)
df[['key_1','key_2','key_3','key_4','key_5','key_6','key_7','key_8','key_9','key_10','key_11']] = \
    df[['key_1','key_2','key_3','key_4','key_5','key_6','key_7','key_8','key_9','key_10','key_11']].astype(int)

df = pd.get_dummies(df, columns=["time_signature"], drop_first=True)
df[['time_signature_4','time_signature_5','time_signature_6','time_signature_7']] = \
    df[['time_signature_4','time_signature_5','time_signature_6','time_signature_7']].astype(int)


model_cols = [col for col in df.columns if col not in cat_but_car]

#######################################################################
# 3. Base Model
#######################################################################


# Standartlaştırma
X_scaled = StandardScaler().fit_transform(df[num_cols])
temp_df = df.copy()
temp_df[num_cols] = pd.DataFrame(X_scaled, columns=num_cols, index=df[num_cols].index)
df = temp_df.copy()


y = df[outcome]

X = df.copy()
X.drop([outcome], axis=1, inplace=True)

for col in cat_but_car:
    X.drop([col], axis=1,inplace=True)


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)


def base_models_rootmse(X, y, scoring="neg_root_mean_squared_error"):
    print("Base Models....")
    regressors = [
        ("XGBoost", XGBRegressor()),
        ("LightGBM", LGBMRegressor(verbose=-1)),
        ("CatBoost", CatBoostRegressor(verbose=False))
    ]
    for name, regressor in regressors:
        cv_results = cross_validate(regressor, X, y, cv=3, scoring=scoring)
        print(f"{scoring}: {round(np.sqrt(-cv_results['test_score'].mean()), 4)} ({name}) ")
    return



base_models_rootmse(X_train, y_train)
# Base Models_rootmse
#neg_root_mean_squared_error: 0.1063 (XGBoost)
#neg_root_mean_squared_error: 0.1073 (LightGBM)
#neg_root_mean_squared_error: 0.1039 (CatBoost)


#######################################################################
# 4. Automated Hyperparameter Optimization
#######################################################################

xgboost_params = {"learning_rate": [0.1],
                  "max_depth": [10, 12, 14],
                  "n_estimators": range(50, 80, 10)}
lightgbm_params = {'learning_rate': [0.01, 0.1, 0.3],
                   'max_depth': [4, 6, 8],
                   'n_estimators': range(50,80,10)}

catboost_params = {'eval_metric': ['RMSE','MAPE'],
                   'iterations': range(500, 1000, 250),
                   'depth': [4, 6, 8, 12]
}


regressors_hpo = [
    ("XGBoost", XGBRegressor(), xgboost_params),
    ("LightGBM", LGBMRegressor(verbose=-1), lightgbm_params),
    ("CatBoost", CatBoostRegressor(verbose=False), catboost_params)
]


def hyperparameter_optimization_rmse(X, y, cv=4, scoring="neg_root_mean_squared_error"):
    print("Hyperparameter Optimization....")
    best_models = {}
    for name, regressor, params in regressors_hpo:
        print(f"########## {name} ##########")
        cv_results = cross_validate(regressor, X, y, cv=cv, scoring=scoring)
        print(f"{scoring} (Before):  {-round(cv_results['test_score'].mean(),4)}")

        gs_best = GridSearchCV(regressor, params, cv=cv, n_jobs=-1, verbose=False).fit(X,y)
        final_model = regressor.set_params(**gs_best.best_params_)

        cv_results = cross_validate(final_model, X, y, cv=cv, scoring=scoring)
        print(f"{scoring} (After): {-round(cv_results['test_score'].mean(),4)}")

        print(f"{name} best params: {gs_best.best_params_}", end="\n\n")
        best_models[name] = final_model
    return best_models


best_models = hyperparameter_optimization_rmse(X_train, y_train, scoring='neg_root_mean_squared_error')

#Hyperparameter Optimization....
########## XGBoost ##########
#neg_root_mean_squared_error (Before):  0.1052
#neg_root_mean_squared_error (After): 0.1042
#XGBoost best params: {'learning_rate': 0.1, 'max_depth': 10, 'n_estimators': 300}


best_xgb_model = XGBRegressor(learning_rate=0.1,max_depth=10,n_estimators=300)


best_xgb_model.fit(X_train,y_train)
y_pred = best_xgb_model.predict(X_test)

mse = mean_squared_error(y_test, y_pred, squared=False) #rmse hesabı
print("Root Mean Squared Error:", mse)
# Mean Squared Error: 0.10400572469934198

mae = mean_absolute_error(y_test, y_pred)
print('Mean Absolute Error:',mae)
# Mean Absolute Error: 0.08132177375680566


#---------------------------------------------Aşağısı çalışma halinde !-----------------------------------
col1 = list(y_test.index)
col2 = y_pred
y_pred_ind = pd.DataFrame(col2, index=col1)

top_200 = y_pred_ind.sort_values(by=0).tail(200).index
y_test.sort_values()

random_50_tracks_ind = random.choices(top_200, k=50)
np.max(random_50_tracks_ind)
np.min(random_50_tracks_ind)
y_pred[14320]
y_test[14320]

y_prd_most = best_xgb_model.predict(pd.DataFrame(X[X.index==66808]))
df[df.index==94275]
df[df.index==top_200]
# random_50_tracks = df.iloc[random_50_tracks_ind[1]]
random_50_tracks = [df_.iloc[indd] for indd in random_50_tracks_ind]

random_50_tracks_ids = [track['track_id'] for track in random_50_tracks]

client_id = '1cc97646ee854447944864d5e0eb3ab8' #XXXTBD sil bunu
client_secret = '06be59d53c2447069aa59f76ad41ee52'

spotipy_add_playlist(client_id, client_secret)

def spotipy_add_playlist(inp_client_id,
                         inp_client_secret,
                         username_id = 11124005204,
                         inp_scope="playlist-modify-public playlist-modify-private"):
    """
    :param username_id:
    :return:
    """

    sp = spotipy.Spotify(auth_manager=SpotifyOAuth(client_id=inp_client_id,
                                                   client_secret= inp_client_secret,
                                                   redirect_uri='https://open.spotify.com/',
                                                   scope=inp_scope))

    hour_now = datetime.now().hour
    minute_now = datetime.now().minute
    playlist_name = "ML Dance Playlist "+str(hour_now)+"_"+str(minute_now)+"_time"
    playlist_description = "This is my new Dance playlist"

    playlist = sp.user_playlist_create(user=username_id, name=playlist_name, public=True,
                                       description=playlist_description)

    # Add tracks to the playlist
    # track_uris = ["spotify:track:4iV5W9uYEdYUVa79Axb7Rh", "spotify:track:2takcwOaAZWiXQijPHIx7B"]
    track_uris = random_50_tracks_ids

    sp.playlist_add_items(playlist_id=playlist["id"], items=track_uris)

    return



import gradio as gr

def greet_user(spotify_userid):
    return "Hello " + name + (" Welcome to Spotify ML Dance Playlist!😎 \n \
    Enter your spotify_userid, spotify_clientid(from spotify for developers), \
                              spotify_secret(from spotify for developers)")

app = gr.Interface(fn = greet_user, inputs="text", outputs="text")

app.launch()