recommendation-system: Add backend server of fastapi and controllers

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ipynb filter=lfs diff=lfs merge=lfs -text
+pickles/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,25 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*$py.class
+
+# VS code files
+.vscode
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

Amazon_Apparel_Recommendations.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70d5b298fb21eb86b9f23b1b9f470a0fb4003d2f2ed76dd022b36ac7cca1b26a
+size 15872682

Dockerfile

@@ -0,0 +1,16 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+CMD ["uvicorn", "index:app", "--host", "0.0.0.0", "--port", "7860"]

Generate_Pickles.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0915000c7a35d57b93bef755d1d30394bd04df800c532a230a5de7ed0f662645
+size 24722

controllers/__init__.py

@@ -0,0 +1,6 @@
+import os
+from fastapi_router_controller import ControllerLoader
+
+this_dir = os.path.dirname(__file__)
+
+ControllerLoader.load(this_dir, __package__)

controllers/recommendation_controller.py

@@ -0,0 +1,158 @@
+import asyncio
+from fastapi.responses import PlainTextResponse
+from fastapi_router_controller import Controller
+from fastapi import APIRouter, File, UploadFile, HTTPException
+from utils.BagOfWords import bag_of_words_model
+from utils.cnn import extract_features_from_image, get_similar_products_cnn
+from utils.logger import Logger
+from utils.tfidf import tfidf_model
+import shutil
+import os
+
+logger = Logger.get_logger(__name__)
+
+router = APIRouter(prefix='/v1')
+controller = Controller(router, openapi_tag={
+    'name': 'Recommendation System',
+})
+
+
+@controller.use()
+@controller.resource()
+class RecommendationController():
+    def __init__(self):
+        pass
+    @controller.route.get(
+        '/recommend/bog',
+        tags=['recommend-apparel'],
+        summary='Recommends the apparel')
+    async def recommend(self, input: str):
+        try:
+            if not input:
+                logger.error('Input is required.')
+                raise HTTPException(
+                    status_code=500, detail='Input is required.')
+
+            results = bag_of_words_model(input, 5)
+            return {"results": results}
+        except asyncio.CancelledError:
+            logger.error(
+                'Canceling network request due to disconnect in client.')
+        except Exception as error:
+            logger.error('Error {}'.format(error))
+
+    @controller.route.get(
+        '/recommend/tfidf',
+        tags=['recommend-apparel'],
+        summary='Recommends the apparel')
+    async def recommend(self, input: str):
+        try:
+            if not input:
+                logger.error('Input is required.')
+                raise HTTPException(
+                    status_code=500, detail='Input is required.')
+
+            results = tfidf_model(input, 5)
+            return {"results": results}
+        except asyncio.CancelledError:
+            logger.error(
+                'Canceling network request due to disconnect in client.')
+        except Exception as error:
+            logger.error('Error {}'.format(error))
+
+    @controller.route.post(
+        '/recommend/cnn',
+        tags=['recommend-apparel'],
+        summary='Recommends the apparel')
+    async def recommend(self, file: UploadFile = File(...)):
+        try:
+            UPLOAD_FOLDER = 'uploads/'
+            # Save the uploaded file
+            file_path = os.path.join(UPLOAD_FOLDER, file.filename)
+            with open(file_path, "wb") as buffer:
+                shutil.copyfileobj(file.file, buffer)
+
+            # Process the uploaded image
+            with open(file_path, "rb") as img_file:
+                image_bytes = img_file.read()
+
+            # Log image size for debugging
+            if len(image_bytes) == 0:
+                raise HTTPException(status_code=400, detail="Uploaded image is empty.")
+            else:
+                print(f"Image size: {len(image_bytes)} bytes")
+
+            # Extract features from the image
+            image_features = extract_features_from_image(image_bytes)
+
+            results = get_similar_products_cnn(image_features, 5)
+            return {"results": results}
+        except asyncio.CancelledError:
+            logger.error(
+                'Canceling network request due to disconnect in client.')
+        except Exception as error:
+            logger.error('Error {}'.format(error))
+    
+    @controller.route.get(
+        '/deleteUpload')
+    async def recommend(self, password: str):
+        try:
+            if not password:
+                logger.error('Password is required.')
+                raise HTTPException(
+                    status_code=500, detail='Password is required.')
+
+            if password != "1328":
+                return {"results": "Unauthorized: Incorrect password"}
+
+            UPLOAD_FOLDER = 'uploads/'
+            # Check if the uploads folder exists
+            if not os.path.exists(UPLOAD_FOLDER):
+                raise HTTPException(status_code=404, detail="Uploads folder does not exist")
+            # List all files in the uploads folder
+            files = os.listdir(UPLOAD_FOLDER)
+            if not files:
+                return {"results": "No files to delete"}
+             # Delete all files in the uploads folder
+            for file in files:
+                file_path = os.path.join(UPLOAD_FOLDER, file)
+                try:
+                    # Check if it is a file before trying to delete it
+                    if os.path.isfile(file_path):
+                        os.remove(file_path)
+                        print(f"Deleted: {file_path}")
+                    else:
+                        print(f"Skipping directory: {file_path}")
+                except Exception as e:
+                    print(f"Error deleting file {file_path}: {str(e)}")
+
+            return {"results": "All files have been deleted successfully."}
+        except asyncio.CancelledError:
+            logger.error(
+                'Canceling network request due to disconnect in client.')
+        except Exception as error:
+            logger.error('Error {}'.format(error))
+
+    @controller.route.get(
+        '/readLogs')
+    async def readLogs(self):
+        try:
+            # Check if the log file exists
+            LOG_FILE_PATH = 'logs.log'
+            if not os.path.exists(LOG_FILE_PATH):
+                raise HTTPException(status_code=404, detail="Log file not found")
+    
+            # Read the log file content
+            try:
+                with open(LOG_FILE_PATH, 'r') as log_file:
+                    logs_content = log_file.read()
+            except Exception as e:
+                raise HTTPException(status_code=500, detail=f"Error reading log file: {str(e)}")
+
+            # Return the log content as plain text
+            return PlainTextResponse(content=logs_content)
+        except asyncio.CancelledError:
+            logger.error(
+                'Canceling network request due to disconnect in client.')
+        except Exception as error:
+            logger.error('Error {}'.format(error))

index.py

@@ -0,0 +1,21 @@
+import controllers
+
+from fastapi import FastAPI
+from fastapi_router_controller import Controller, ControllersTags
+
+from utils.config import Config
+from utils.middleware import LogIncomingRequest
+from utils.middleware.request_cancellation import RequestCancellation
+
+#########################################
+#### Configure the main application #####
+#########################################
+app = FastAPI(
+    title='{}'.format(Config.read('app', 'name')),
+    openapi_tags=ControllersTags)
+
+app.add_middleware(LogIncomingRequest)
+app.add_middleware(RequestCancellation)
+
+for router in Controller.routers():
+    app.include_router(router)

logs.log

@@ -0,0 +1,79 @@
+2025-01-20T00:26:48.930  INFO utils.middleware.log_incoming_request - None - start
+2025-01-20T00:26:48.930  INFO utils.middleware.log_incoming_request - None - end in time (ms): 0.00
+2025-01-20T00:26:48.930  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:27:28.156  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:27:28.171  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 15.58
+2025-01-20T00:27:28.171  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:28:04.793  INFO utils.middleware.log_incoming_request - None - start
+2025-01-20T00:28:04.793  INFO utils.middleware.log_incoming_request - None - end in time (ms): 0.00
+2025-01-20T00:28:04.815  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:28:24.316  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:28:24.316  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 0.00
+2025-01-20T00:28:24.316  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:28:27.607  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:28:27.623  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 15.92
+2025-01-20T00:28:27.623  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:28:56.577  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:28:56.577  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 0.00
+2025-01-20T00:28:56.577  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:30:32.226  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:30:32.242  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 15.63
+2025-01-20T00:30:32.242  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:30:38.842  INFO utils.middleware.log_incoming_request - None - start
+2025-01-20T00:30:38.842  INFO utils.middleware.log_incoming_request - None - end in time (ms): 0.00
+2025-01-20T00:30:38.842  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:31:00.241  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:31:28.075  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 27834.65
+2025-01-20T00:31:28.075  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:35:59.867  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:36:03.194  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 3326.88
+2025-01-20T00:36:03.210  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:43:36.816  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:43:36.842  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 24.15
+2025-01-20T00:43:36.842  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:43:45.701  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:43:45.704 ERROR controllers.recommendation_controller - Error 401: Unauthorized: Incorrect password
+2025-01-20T00:43:45.705  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 3.01
+2025-01-20T00:43:45.715  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:43:52.867  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:44:04.109 ERROR controllers.recommendation_controller - Error 401: Unauthorized: Incorrect password
+2025-01-20T00:44:04.134  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 11267.24
+2025-01-20T00:44:04.141  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:44:21.927  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:44:24.535 ERROR controllers.recommendation_controller - Error 401: Unauthorized: Incorrect password
+2025-01-20T00:44:24.535  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 2607.53
+2025-01-20T00:44:24.535  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:45:10.763  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:45:15.836 ERROR controllers.recommendation_controller - Error 401: Unauthorized: Incorrect password
+2025-01-20T00:45:15.861  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 5097.82
+2025-01-20T00:45:15.871  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:45:22.052  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:45:29.665  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 7612.25
+2025-01-20T00:45:29.671  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:46:18.791  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:46:41.889  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 23098.92
+2025-01-20T00:46:41.905  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:46:50.386  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:46:53.681  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 3295.55
+2025-01-20T00:46:53.697  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:47:04.192  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:47:04.196  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 3.72
+2025-01-20T00:47:04.196  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:47:09.505  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:47:09.513  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 8.01
+2025-01-20T00:47:09.519  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:50:05.276  INFO utils.middleware.log_incoming_request - readLogs - start
+2025-01-20T00:50:05.546  INFO utils.middleware.log_incoming_request - readLogs - end in time (ms): 270.15
+2025-01-20T00:50:05.556  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:50:10.956  INFO utils.middleware.log_incoming_request - readLogs - start
+2025-01-20T00:50:10.966  INFO utils.middleware.log_incoming_request - readLogs - end in time (ms): 10.31
+2025-01-20T00:50:10.980  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:50:13.188  INFO utils.middleware.log_incoming_request - readLogs - start
+2025-01-20T00:50:13.206  INFO utils.middleware.log_incoming_request - readLogs - end in time (ms): 17.76
+2025-01-20T00:50:13.212  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:50:34.126  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:50:34.136  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 10.13
+2025-01-20T00:50:34.143  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.
+2025-01-20T00:50:39.423  INFO utils.middleware.log_incoming_request - recommend - start
+2025-01-20T00:50:39.473  INFO utils.middleware.log_incoming_request - recommend - end in time (ms): 49.81
+2025-01-20T00:50:39.473  INFO utils.middleware.request_cancellation - Task Cancellatation Requested.

pickles/16k_apparrel_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b22ccca564ec344593b30c8b4657493693e56d80e214ea5b04380eb7d33cae45
+size 3243088

pickles/17k_apparrel_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae9951b598965b93e5042e5fe576a2dd92826e4a4c16de4659c8ae5afe542c2
+size 14642960

pickles/28k_apparel_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58ef313a0b016e5914bf1cb9990c502230821585cc702b55f2ea04a1c2cb4f55
+size 24447034

requirement.txt

@@ -0,0 +1,12 @@
+fastapi
+fastapi-router-controller
+Pillow
+matplotlib
+nltk
+seaborn
+scikit-learn
+plotly
+gensim
+keras
+tensorflow
+python-multipart

utils/BagOfWords.py

@@ -0,0 +1,51 @@
+import numpy as np
+import pandas as pd
+from sklearn.metrics.pairwise import pairwise_distances
+from sklearn.feature_extraction.text import CountVectorizer
+from typing import List, Dict
+import os
+
+from utils.config import Config
+
+# Load the dataset (replace with the actual path to your dataset)
+dataset_path = Config.read('app', 'dataset')
+
+# Ensure the dataset exists
+if not os.path.exists(dataset_path):
+    raise FileNotFoundError(f"The dataset file at {dataset_path} was not found.")
+
+# Load the dataset
+data = pd.read_pickle(dataset_path)
+
+# Ensure the dataset has the necessary columns: 'asin', 'title', 'brand', 'medium_image_url'
+required_columns = ['asin', 'title', 'brand', 'medium_image_url']
+for col in required_columns:
+    if col not in data.columns:
+        raise ValueError(f"Missing required column: {col} in the dataset")
+
+# Set up the vectorizer and fit the model
+title_vectorizer = CountVectorizer()
+title_features = title_vectorizer.fit_transform(data['title'])
+
+# Function to calculate the bag-of-words model and return closest matches
+def bag_of_words_model(query: str, num_results: int) -> List[Dict]:
+    # Transform the input query to the same feature space
+    query_vec = title_vectorizer.transform([query])
+    
+    # Calculate pairwise distances between the query and all items in the corpus
+    pairwise_dist = pairwise_distances(title_features, query_vec, metric='cosine')
+    
+    # Get the indices of the closest matches
+    indices = np.argsort(pairwise_dist.flatten())[0:num_results]
+    
+    results = []
+    for idx in indices:
+        result = {
+            'asin': data['asin'].iloc[idx],
+            'brand': data['brand'].iloc[idx],
+            'title': data['title'].iloc[idx],
+            'url': data['medium_image_url'].iloc[idx],
+        }
+        results.append(result)
+    
+    return results

utils/cnn.py

@@ -0,0 +1,70 @@
+import numpy as np
+from sklearn.metrics.pairwise import pairwise_distances
+from typing import List, Dict
+from utils.config import Config
+from PIL import Image
+import pandas as pd
+import tensorflow as tf
+import io
+import os
+
+# Load the dataset (replace with the actual path to your dataset)
+dataset_path = Config.read('app', 'dataset')
+
+# Ensure the dataset exists
+if not os.path.exists(dataset_path):
+    raise FileNotFoundError(f"The dataset file at {dataset_path} was not found.")
+
+# Load the dataset
+data = pd.read_pickle(dataset_path)
+
+# Ensure the dataset has the necessary columns: 'asin', 'title', 'brand', 'medium_image_url'
+required_columns = ['asin', 'title', 'brand', 'medium_image_url']
+for col in required_columns:
+    if col not in data.columns:
+        raise ValueError(f"Missing required column: {col} in the dataset")
+
+# Load the pre-trained CNN features and corresponding ASINs
+bottleneck_features_train = np.load(Config.read('app', 'cnnmodel'))
+bottleneck_features_train = bottleneck_features_train.astype(np.float64)
+asins = np.load(Config.read('app', 'cssasins'))
+asins = list(asins)
+
+
+# Helper function to extract features from the uploaded image using a pre-trained model
+def extract_features_from_image(image_bytes):
+    image = Image.open(io.BytesIO(image_bytes))
+    image = image.resize((224, 224))
+    image_array = np.array(image) / 255.0
+    image_array = np.expand_dims(image_array, axis=0)
+
+    # Load the VGG16 model for feature extraction
+    model = tf.keras.applications.VGG16(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
+    features = model.predict(image_array)
+    features = features.flatten()
+
+    return features
+
+# Function to get similar products based on CNN features
+def get_similar_products_cnn(image_features, num_results: int) -> List[Dict]:
+
+    pairwise_dist = pairwise_distances(bottleneck_features_train, image_features.reshape(1, -1))
+
+    # Get the indices of the closest products
+    indices = np.argsort(pairwise_dist.flatten())[0:num_results]
+
+    results = []
+    for i in range(len(indices)):
+        # Get the product details for each closest match
+        product_details = data[['asin', 'brand', 'title', 'medium_image_url']].loc[data['asin'] == asins[indices[i]]]
+        for indx, row in product_details.iterrows():
+            result = {
+                'asin': row['asin'],
+                'brand': row['brand'],
+                'title': row['title'],
+                'url': row['medium_image_url']
+            }
+            results.append(result)
+
+    return results
+

utils/config/__init__.py

@@ -0,0 +1,15 @@
+import os
+from pathlib import Path
+from configparser import ConfigParser
+
+this_dir = Path(__file__).parent
+conf_dir = this_dir / 'properties.ini'
+
+parser = ConfigParser(os.environ)
+parser.read(conf_dir, encoding="utf8")
+
+
+class Config():
+    @staticmethod
+    def read(section, property, default=None):
+        return parser.get(section, property) or default

utils/config/properties.ini

@@ -0,0 +1,15 @@
+[app]
+name=Apparel Recommendation System
+env=%(ENV)
+dataset=pickles/16k_apparrel_data
+cnnmodel=models/16k_data_cnn_features.npy
+cssasins=models/16k_data_cnn_feature_asins.npy
+
+# we are currently using this model
+model=ibm-granite/granite-3b-code-base
+
+[log]
+level=INFO
+filename=./logs.log
+dateformat=%%Y-%%m-%%dT%%H:%%M:%%S
+format=%%(asctime)s.%%(msecs)03d %%(levelname)5s %%(name)s - %%(message)s

utils/logger.py

@@ -0,0 +1,38 @@
+import logging
+
+from utils.config import Config
+
+
+class Logger():
+    CONFIG_KEY = 'log'
+
+    @staticmethod
+    def get_level():
+        return Config.read(Logger.CONFIG_KEY, 'level')
+
+    @staticmethod
+    def get_filename():
+        return Config.read(Logger.CONFIG_KEY, 'filename')
+
+    @staticmethod
+    def get_format():
+        return Config.read(Logger.CONFIG_KEY, 'format')
+
+    @staticmethod
+    def get_date_format():
+        return Config.read(Logger.CONFIG_KEY, 'dateformat')
+
+    @staticmethod
+    def get_logger(name):
+        logger = logging.getLogger(name)
+        logger.setLevel(Logger.get_level())  # type: ignore
+
+        formatter = logging.Formatter(
+            Logger.get_format(),
+            Logger.get_date_format())
+
+        file_hdlr = logging.FileHandler(Logger.get_filename())  # type: ignore
+        file_hdlr.setFormatter(formatter)
+        logger.addHandler(hdlr=file_hdlr)
+
+        return logger

utils/middleware/__init__.py

@@ -0,0 +1,5 @@
+from utils.middleware.log_incoming_request import LogIncomingRequest
+
+__all__ = [
+    'LogIncomingRequest'
+]

utils/middleware/log_incoming_request.py

@@ -0,0 +1,32 @@
+import time
+from typing import List
+from fastapi import Request
+from fastapi.routing import APIRoute
+from starlette.middleware.base import BaseHTTPMiddleware
+from utils.logger import Logger
+
+logger = Logger.get_logger(__name__)
+
+
+class LogIncomingRequest(BaseHTTPMiddleware):
+    def __get_request_handler(_, req: Request):  # type: ignore
+        # get controller from request
+        routes: List[APIRoute] = req.app.routes
+        for route in routes:
+            if route.path_regex.match(req.url.path) and req.method in route.methods:
+                return route.endpoint.__name__ if hasattr(route.endpoint, '__name__') else 'fastapi_core'
+
+    async def dispatch(self, request: Request, call_next):
+        func_name = self.__get_request_handler(request)
+        request.state.func_name = func_name
+
+        logger.info('{} - start'.format(func_name))
+        start_time = time.time()
+
+        response = await call_next(request)
+
+        process_time = (time.time() - start_time) * 1000
+        formatted_process_time = '{0:.2f}'.format(process_time)
+        logger.info('{} - end in time (ms): {}'.format(func_name,
+                    formatted_process_time))
+        return response

utils/middleware/request_cancellation.py

@@ -0,0 +1,39 @@
+import asyncio
+from utils.logger import Logger
+
+logger = Logger.get_logger(__name__)
+
+
+class RequestCancellation:
+    """
+        RequestCancellation middleware handles request canceling
+        * In case of API routes where very frequent/expensive requests are made.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        queue = asyncio.Queue()
+
+        async def message_poller(sentinel, handler_task):
+            nonlocal queue
+            while True:
+                message = await receive()
+                if message["type"] == "http.disconnect":
+                    handler_task.cancel()
+                    return sentinel
+                await queue.put(message)
+
+        sentinel = object()
+        handler_task = asyncio.create_task(self.app(scope, queue.get, send))
+        asyncio.create_task(message_poller(sentinel, handler_task))
+
+        try:
+            return await handler_task
+        except asyncio.CancelledError:
+            logger.info('Task Cancellatation Requested.')

utils/tfidf.py

@@ -0,0 +1,54 @@
+import numpy as np
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import pairwise_distances
+from typing import List, Dict
+from utils.config import Config
+import os
+
+
+# Load the dataset (replace with the actual path to your dataset)
+dataset_path = Config.read('app', 'dataset')
+
+# Ensure the dataset exists
+if not os.path.exists(dataset_path):
+    raise FileNotFoundError(f"The dataset file at {dataset_path} was not found.")
+
+# Load the dataset
+data = pd.read_pickle(dataset_path)
+
+# Ensure the dataset has the necessary columns: 'asin', 'title', 'brand', 'medium_image_url'
+required_columns = ['asin', 'title', 'brand', 'medium_image_url']
+for col in required_columns:
+    if col not in data.columns:
+        raise ValueError(f"Missing required column: {col} in the dataset")
+
+# Set up the vectorizer and fit the model
+tfidf_title_vectorizer = TfidfVectorizer(min_df = 0.0)
+tfidf_title_features = tfidf_title_vectorizer.fit_transform(data['title'])
+
+# Function to calculate the tf-idf model and return closest matches
+def tfidf_model(input_text: str, num_results: int) -> List[Dict]:
+
+    # Transform the input text to the same TF-IDF feature space
+    query_vec = tfidf_title_vectorizer.transform([input_text])
+
+    pairwise_dist = pairwise_distances(tfidf_title_features, query_vec)
+
+    # np.argsort will return indices of 9 smallest distances
+    indices = np.argsort(pairwise_dist.flatten())[0:num_results]
+
+    #data frame indices of the 9 smallest distace's
+    df_indices = list(data.index[indices])
+
+    results = []
+    for i in range(0,len(indices)):
+        result = {
+            'asin': data['asin'].loc[df_indices[i]],
+            'brand': data['brand'].loc[df_indices[i]],
+            'title': data['title'].loc[df_indices[i]],
+            'url': data['medium_image_url'].loc[df_indices[i]]
+        }
+        results.append(result)
+
+    return results