all my project

commands.txt

@@ -0,0 +1,5 @@
+python3 -m venv env_name
+# activate
+
+# install dependencies
+pip install -r requirements.txt

requirements.txt

@@ -0,0 +1,6 @@
+gradio==3.23.0
+pandas==1.5.3
+pytest
+# ML DEV
+scikit-learn==1.2.2
+pandas-profiling==3.6.6

src/app.py

@@ -0,0 +1,142 @@
+import gradio as gr
+import pandas as pd
+import os
+import pickle
+from datetime import datetime
+
+
+# PAGE CONFIG
+# page_icon = "💐"
+
+# Setup variables and constants
+# datetime.now().strftime('%d-%m-%Y _ %Hh %Mm %Ss')
+DIRPATH = os.path.dirname(os.path.realpath(__file__))
+tmp_df_fp = os.path.join(DIRPATH, "assets", "tmp",
+                         f"history_{datetime.now().strftime('%d-%m-%Y')}.csv")
+ml_core_fp = os.path.join(DIRPATH, "assets", "ml", "ml_components.pkl")
+init_df = pd.DataFrame(
+    {"petal length (cm)": [], "petal width (cm)": [],
+     "sepal length (cm)": [], "sepal width (cm)": [], }
+)
+
+# FUNCTIONS
+
+
+def load_ml_components(fp):
+    "Load the ml component to re-use in app"
+    with open(fp, "rb") as f:
+        object = pickle.load(f)
+    return object
+
+
+def setup(fp):
+    "Setup the required elements like files, models, global variables, etc"
+
+    # history frame
+    if not os.path.exists(fp):
+        df_history = init_df.copy()
+    else:
+        df_history = pd.read_csv(fp)
+
+    df_history.to_csv(fp, index=False)
+
+    return df_history
+
+
+def make_prediction(df_input):
+    """Function that take a dataframe as input and make prediction
+    """
+    global df_history
+
+    print(f"\n[Info] Input information as dataframe: \n{df_input.to_string()}")
+    df_input.drop_duplicates(inplace=True, ignore_index=True)
+    print(f"\n[Info] Input with deplicated rows: \n{df_input.to_string()}")
+
+    prediction_output = end2end_pipeline.predict_proba(df_input)
+    print(
+        f"[Info] Prediction output (of type '{type(prediction_output)}') from passed input: {prediction_output} of shape {prediction_output.shape}")
+
+    predicted_idx = prediction_output.argmax(axis=-1)
+    print(f"[Info] Predicted indexes: {predicted_idx}")
+    df_input['pred_label'] = predicted_idx
+    print(
+        f"\n[Info] pred_label: \n{df_input.to_string()}")
+    predicted_labels = df_input['pred_label'].replace(idx_to_labels)
+    df_input['pred_label'] = predicted_labels
+    print(
+        f"\n[Info] convert pred_label: \n{df_input.to_string()}")
+
+    predicted_score = prediction_output.max(axis=-1)
+    print(f"\n[Info] Prediction score: \n{predicted_score}")
+    df_input['confidence_score'] = predicted_score
+    print(
+        f"\n[Info] output information as dataframe: \n{df_input.to_string()}")
+    df_history = pd.concat([df_history, df_input], ignore_index=True).drop_duplicates(
+        ignore_index=True, keep='last')
+    return df_history
+
+
+def download():
+    return gr.File.update(label="History File",
+                          visible=True,
+                          value=tmp_df_fp)
+
+
+def hide_download():
+    return gr.File.update(label="History File",
+                          visible=False)
+
+
+# Setup execution
+ml_components_dict = load_ml_components(fp=ml_core_fp)
+labels = ml_components_dict['labels']
+end2end_pipeline = ml_components_dict['pipeline']
+print(f"\n[Info] ML components loaded: {list(ml_components_dict.keys())}")
+print(f"\n[Info] Predictable labels: {labels}")
+idx_to_labels = {i: l for (i, l) in enumerate(labels)}
+print(f"\n[Info] Indexes to labels: {idx_to_labels}")
+
+df_history = setup(tmp_df_fp)
+
+
+# APP Interface
+with gr.Blocks() as demo:
+    gr.Markdown('''<img class="center" src="https://www.thespruce.com/thmb/GXt55Sf9RIzADYAG5zue1hXtlqc=/1500x0/filters:no_upscale():max_bytes(150000):strip_icc()/iris-flowers-plant-profile-5120188-01-04a464ab8523426fab852b55d3bb04f0.jpg"  width="50%" height="50%"> 
+    <style>
+    .center {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    width: 50%;
+    }
+    </style>''')
+    gr.Markdown('''# 💐 Iris Classification App
+    This app shows a simple demo of a Gradio app for Iris flowers classification.
+    ''')
+
+    df = gr.Dataframe(
+        headers=["petal length (cm)",
+                 "petal width (cm)",
+                 "sepal length (cm)",
+                 "sepal width (cm)"],
+        datatype=["number", "number", "number", "number", ],
+        row_count=3,
+        col_count=(4, "fixed"),
+    )
+    output = gr.Dataframe(df_history)
+
+    btn_predict = gr.Button("Predict")
+    btn_predict.click(fn=make_prediction, inputs=df, outputs=output)
+
+    # output.change(fn=)
+
+    file_obj = gr.File(label="History File",
+                       visible=False
+                       )
+
+    btn_download = gr.Button("Download")
+    btn_download.click(fn=download, inputs=[], outputs=file_obj)
+    output.change(fn=hide_download, inputs=[], outputs=file_obj)
+
+if __name__ == "__main__":
+    demo.launch(debug=True)

src/assets/ml/requirements.txt

@@ -0,0 +1,85 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==22.2.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+entrypoints==0.4
+exceptiongroup==1.1.1
+fastapi==0.95.0
+ffmpy==0.3.0
+filelock==3.10.7
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.3.0
+gradio==3.23.0
+h11==0.14.0
+htmlmin==0.1.12
+httpcore==0.16.3
+httpx==0.23.3
+huggingface-hub==0.13.3
+idna==3.4
+ImageHash==4.3.1
+iniconfig==2.0.0
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.6.3
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+multimethod==1.9.1
+networkx==3.0
+numpy==1.23.5
+orjson==3.8.9
+packaging==23.0
+pandas==1.5.3
+pandas-profiling==3.6.6
+patsy==0.5.3
+phik==0.12.3
+Pillow==9.4.0
+pluggy==1.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+pytest==7.2.2
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.2
+PyWavelets==1.4.1
+PyYAML==6.0
+requests==2.28.2
+rfc3986==1.5.0
+scikit-learn==1.2.2
+scipy==1.9.3
+seaborn==0.12.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+statsmodels==0.13.5
+tangled-up-in-unicode==0.2.0
+threadpoolctl==3.1.0
+tomli==2.0.1
+toolz==0.12.0
+tqdm==4.64.1
+typeguard==2.13.3
+typing_extensions==4.5.0
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+visions==0.7.5
+websockets==10.4
+yarl==1.8.2
+ydata-profiling==4.1.2

src/problem_solving.py

@@ -0,0 +1,139 @@
+# Imports
+
+import pickle
+import os
+from sklearn.metrics import classification_report, ConfusionMatrixDisplay
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.impute import SimpleImputer, KNNImputer
+from sklearn.pipeline import Pipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import StandardScaler, RobustScaler, OneHotEncoder
+from sklearn.model_selection import train_test_split
+import pandas as pd
+from ydata_profiling import ProfileReport
+from sklearn import datasets
+from subprocess import call
+
+# PATHS
+DIRPATH = os.path.dirname(os.path.realpath(__file__))
+ml_fp = os.path.join(DIRPATH, "assets", "ml", "ml_components.pkl")
+req_fp = os.path.join(DIRPATH, "assets", "ml", "requirements.txt")
+eda_report_fp = os.path.join(DIRPATH, "assets", "ml", "eda-report.html")
+
+# import some data to play with
+iris = datasets.load_iris(return_X_y=False, as_frame=True)
+
+df = iris['frame']
+target_col = 'target'
+# pandas profiling
+profile = ProfileReport(df, title="Dataset", html={
+                        'style': {'full_width': True}})
+profile.to_file(eda_report_fp)
+
+# Dataset Splitting
+# Please specify
+to_ignore_cols = [
+    "ID",  # ID
+    "Id", "id",
+    target_col
+]
+
+
+num_cols = list(set(df.select_dtypes('number')) - set(to_ignore_cols))
+cat_cols = list(set(df.select_dtypes(exclude='number')) - set(to_ignore_cols))
+print(f"\n[Info] The '{len(num_cols)}' numeric columns are : {num_cols}\nThe '{len(cat_cols)}' categorical columns are : {cat_cols}")
+
+X, y = df.iloc[:, :-1], df.iloc[:, -1].values
+
+
+X_train, X_eval, y_train, y_eval = train_test_split(
+    X, y, test_size=0.2, random_state=0, stratify=y)
+
+print(
+    f"\n[Info] Dataset splitted : (X_train , y_train) = {(X_train.shape , y_train.shape)}, (X_eval y_eval) = {(X_eval.shape , y_eval.shape)}. \n")
+
+y_train
+
+# Modeling
+
+# Imputers
+num_imputer = SimpleImputer(strategy="mean").set_output(transform="pandas")
+cat_imputer = SimpleImputer(
+    strategy="most_frequent").set_output(transform="pandas")
+
+# Scaler & Encoder
+if len(cat_cols) > 0:
+    df_imputed_stacked_cat = cat_imputer.fit_transform(
+        df
+        .append(df)
+        .append(df)
+        [cat_cols])
+    cat_ = OneHotEncoder(sparse=False, drop="first").fit(
+        df_imputed_stacked_cat).categories_
+else:
+    cat_ = 'auto'
+
+encoder = OneHotEncoder(categories=cat_, sparse=False, drop="first")
+scaler = StandardScaler().set_output(transform="pandas")
+
+
+# feature pipelines
+num_pipe = Pipeline(steps=[("num_imputer", num_imputer), ("scaler", scaler)])
+cat_pipe = Pipeline(steps=[("cat_imputer", cat_imputer), ("encoder", encoder)])
+
+# end2end features preprocessor
+
+transformers = []
+
+transformers.append(("numerical", num_pipe, num_cols)) if len(
+    num_cols) > 0 else None
+transformers.append(("categorical", cat_pipe, cat_cols,)) if len(
+    cat_cols) > 0 else None
+#  ("date", date_pipe, date_cols,),
+
+preprocessor = ColumnTransformer(
+    transformers=transformers).set_output(transform="pandas")
+
+print(
+    f"\n[Info] Features Transformer : {transformers}. \n")
+
+
+# end2end pipeline
+end2end_pipeline = Pipeline([
+    ('preprocessor', preprocessor),
+    ('model', RandomForestClassifier(random_state=10))
+]).set_output(transform="pandas")
+
+# Training
+print(
+    f"\n[Info] Training.\n[Info] X_train : columns( {X_train.columns.tolist()}), shape: {X_train.shape} .\n")
+
+end2end_pipeline.fit(X_train, y_train)
+
+# Evaluation
+print(
+    f"\n[Info] Evaluation.\n")
+y_eval_pred = end2end_pipeline.predict(X_eval)
+
+print(classification_report(y_eval, y_eval_pred,
+      target_names=iris['target_names']))
+
+# ConfusionMatrixDisplay.from_predictions(
+#     y_eval, y_eval_pred, display_labels=iris['target_names'])
+
+# Exportation
+print(
+    f"\n[Info] Exportation.\n")
+to_export = {
+    "labels": iris['target_names'],
+    "pipeline": end2end_pipeline,
+}
+
+
+# save components to file
+with open(ml_fp, 'wb') as file:
+    pickle.dump(to_export, file)
+
+# Requirements
+# ! pip freeze > requirements.txt
+call(f"pip freeze > {req_fp}", shell=True)

src/tmp.txt

@@ -0,0 +1,15 @@
+# Setup execution
+ml_components = load_ml_components(fp=ml_core_fp)
+idx_to_labels = {i: l for (i, l) in enumerate(ml_components["labels"])}
+end_2_end_pipeline = ml_components["pipeline"]
+
+
+            # Prediction
+            pred_output = end_2_end_pipeline.predict_proba(df_input)
+
+            class_idx = pred_output.argmax(axis=1)
+            scores = pred_output[:, class_idx]
+            df_input['predicted_class'] = class_idx
+            df_input['predicted_class'] = df_input['predicted_class'].replace(
+                idx_to_labels)
+            df_input['confidence_score'] = scores