Spaces:
Runtime error
Runtime error
| import functions.extract_function as get | |
| import functions.preprocessing_function as preprocess | |
| import functions.modelling_function as modelling | |
| import os | |
| import re | |
| import math | |
| import numpy as np | |
| from rapidfuzz import process, fuzz, utils | |
| from simpletransformers.classification import ClassificationModel | |
| from transformers import pipeline | |
| import gradio as gr | |
| def is_nan(text): | |
| try: | |
| value = float(text) | |
| return math.isnan(value) | |
| except ValueError: | |
| return False | |
| # Function for preparing catalog | |
| def prepare_catalog(): | |
| # Load internal catalog | |
| product_catalog = get.internal_data('catalog') | |
| # Load external catalog | |
| registered_fertilizers = get.registered_fertilizer_data() | |
| # Product catalog cleaning | |
| product_catalog = preprocess.clean_dataframe(product_catalog, 'Product SKU', remove_na=False, remove_non_words=True, remove_symbols=True) | |
| product_catalog['Product SKU Full Clean'] = product_catalog['Product SKU Clean'] + ' ' + product_catalog['Brand'].str.lower() + ' ' + product_catalog['Type'].str.lower() | |
| product_catalog['Product SKU Full'] = product_catalog['Product SKU'] + ' ' + product_catalog['Brand'].str.lower() + ' ' + product_catalog['Type'].str.lower() | |
| # Removing Duplicates: | |
| product_catalog = preprocess.fuzzy_join_compare(product_catalog, 'Product SKU Clean', 'Product SKU Full Clean', registered_fertilizers, take_regist_number=True, set_ratio_weight=1, ratio_weight=0) | |
| # 1. Only take registered fertilizers that is NOT in the existing product catalog | |
| registered_fertilizers = preprocess.slice_with_filter(registered_fertilizers, 'Nomor Pendaftaran', product_catalog, use_filter=True, filter_condition= product_catalog['Max Similarity Score'] > 80) | |
| # 2. Combine product catalog and registered fertilizers | |
| combined_catalog = preprocess.combine_catalog(product_catalog['Product SKU Full'], registered_fertilizers['Nama Lengkap'], 'Product Catalog', 'Registered Fertilizers') | |
| # 3. Remove duplicates | |
| combined_catalog = combined_catalog.drop_duplicates() | |
| # Use lambda function to extract the formula from Registered Product column | |
| combined_catalog['Formula'] = combined_catalog['Registered Product'].apply(lambda x: re.findall(r'\d{1,2}\s*[- ]\s*\d{1,2}\s*[- ]\s*\d{1,2}', x)) | |
| # if formula is empty list, then replace it with NaN, else take the first item in the formula list | |
| combined_catalog['Formula'] = combined_catalog['Formula'].apply(lambda x: np.nan if len(x) == 0 else x[0]) | |
| return combined_catalog | |
| # Your existing decision function | |
| def decision(user_input, catalog, product_name_catalog): | |
| # Initialize the model | |
| pipe = pipeline("text-classification", model="matthewfarant/indobert-fertilizer-classifier", token = os.getenv('HF_MY_TOKEN')) | |
| # Extract formula | |
| user_input_formula = re.findall(r'\d{1,2}\s*[- ]\s*\d{1,2}\s*[- ]\s*\d{1,2}', user_input) | |
| user_input_formula = np.nan if len(user_input_formula) == 0 else user_input_formula[0] | |
| # Similar Product | |
| catalog['Similarity Score'] = catalog[product_name_catalog].apply(lambda x: fuzz.token_set_ratio(user_input, x, processor=utils.default_process)) | |
| catalog['Formula Similarity'] = catalog['Formula'].apply(lambda x: fuzz.token_set_ratio(user_input_formula, x, processor=utils.default_process)) | |
| # Take Top Similar Product. Take "Product Catalog" first | |
| catalog = catalog.sort_values(by=['Similarity Score', 'Formula Similarity', 'Source'], ascending=[False, False, True]).head(1) | |
| # Condition | |
| if catalog['Similarity Score'].values[0] > 80 and catalog['Source'].values[0] == 'Product Catalog' and (catalog['Formula Similarity'].values[0] == 100 or is_nan(catalog['Formula'].values[0])): | |
| return f"Product is Available in Catalog (SKU Registered as {catalog['Registered Product'].values[0]})" | |
| elif catalog['Similarity Score'].values[0] > 80 and catalog['Source'].values[0] == 'Registered Fertilizers' and (catalog['Formula Similarity'].values[0] == 100 or is_nan(catalog['Formula'].values[0])): | |
| return f"Add as New Product (Registered in Kementan as {catalog['Registered Product'].values[0]})" | |
| elif catalog['Similarity Score'].values[0] > 80 and catalog['Formula Similarity'].values[0] < 100: | |
| return f"Add as New Product (Similar to {catalog['Registered Product'].values[0]} in {catalog['Source'].values[0]} but with different formula)" | |
| elif catalog['Similarity Score'].values[0] < 80: | |
| if pipe(user_input)[0]['label'] == 'Fertilizer' and pipe(user_input)[0]['score'] > 0.8: | |
| return f"Add as New Product ({pipe(user_input)[0]['score'] * 100}% probability of being a fertilizer)" | |
| else: | |
| return f"Product might not be a Fertilizer ({np.round(pipe(user_input)[0]['score'] * 100,2)}% probability of being a {pipe(user_input)[0]['label']})" | |
| else: | |
| return "Product is not a Fertilizer" | |
| def app(input): | |
| catalog = prepare_catalog() | |
| return decision(input, catalog, "Registered Product") | |
| # Initialize the app | |
| demo = gr.Interface( | |
| fn=app, | |
| inputs="text", | |
| outputs="text", | |
| examples= ['Petro Nitrat 16-16-16', 'Petro Nitrat 15-15-15', 'Gramoxone 1 Liter', 'Indomie Goreng Aceh'], | |
| title = 'Fertilizer Catalog Engine 🌽', | |
| description = 'Catalog Search Engine and Decision Support System for Fertilizer Company', | |
| article= """ | |
| ### About The App | |
| This app is built as a part of the Data Science Weekend 2023 Challenge submission. This app aims to help fertilizer companies to map | |
| free-text POS data of multiple types of products into their own fertilizer catalog. By using this app, the company will be able to | |
| decide whether a product is already available in their catalog, or whether it is a new product that needs (and eligible) to be added to | |
| the catalog. <br> | |
| ### How Does it Work? | |
| This app uses a combination of fuzzy matching and machine learning to determine whether a product is already available in the catalog or not. | |
| When a product is not available in the catalog, we will use a IndoBERT model to determine if the product is a fertilizer and eligible to be | |
| added to the catalog. Beforehand, we have fine-tuned the IndoBERT model using a combination of internal and external (web scraping) data, so the | |
| model will be able to learn how fertilizer products (especially the local ones) look like. <br> | |
| ### What is the Flags For? | |
| The flag is a part of the Active Transfer Learning feature of this app. When a user flags a product as "Correct" or "Incorrect", the developer | |
| will be able to fine-tune the model using the user's input, hence improving the model's performance. So, please help us to improve the model by | |
| flagging the prediction result! <br> | |
| ### I want to test multiple inputs at once! | |
| You can also use our app via API by clicking the "Use via API" below. The API will give developer more flexibility to test multiple inputs | |
| programmatically. <br> | |
| """, | |
| api_name='search', | |
| flagging_options=["Correct","Incorrect"], | |
| theme = gr.themes.Soft() | |
| ) | |
| # Run the app | |
| if __name__ == "__main__": | |
| demo.launch(show_api=True) |