v2_update

app.py

@@ -1,13 +1,9 @@
-import data_prep
-import model_predict
 import gradio as gr
-
-# model_dict = {
-#     "BERT-Base": "research-dump/bert-base-uncased_deletion_multiclass_complete_Final",
-#     "BERT-Large": "research-dump/bert-large-uncased_deletion_multiclass_complete_final",
-#     "RoBERTa-Base": "research-dump/roberta-base_deletion_multiclass_complete_final",
-#     "RoBERTa-Large": "research-dump/roberta-large_deletion_multiclass_complete_final"
-# }
+import pandas as pd
+import torch
+from data_collect import collect
+from model_predict import predict_text
+from wide_analysis.data.process_data import prepare_dataset
 
 model_dict = {
     "Outcome Prediction": "outcome",
@@ -17,42 +13,173 @@ model_dict = {
     "Offensive Language Detection": "offensive"
 }
 
-def process_url(url, model_name):
-    model_name = model_dict[model_name]
-    processed_text = data_prep.process_data(url)
-    final_scores = model_predict.predict_text(processed_text, model_name)
+platform_dict = ["wikipedia", "wikidata_entity", "wikidata_property", "wikinews", "wikiquote"]
+lang_dict = ["en", "es", "gr"]
+
+title = 'Wide-Analysis: A Wikipedia Deletion Discussion Analysis Suite'
+desc = """ Wide-Analysis is a suite of tools for analyzing deletion discussions across various Wikimedia platforms in multiple languages... """
+
+
+def process_url(url, task_name, lang, platform, date):
+    model_name = model_dict[task_name]
+    derived_date = url.split('/')[-1].split('#')[0]
+    derived_title = url.split('#')[-1]
+
+    if model_name == 'outcome':
+        if lang == 'en' and platform == 'wikipedia':
+            df = prepare_dataset(mode='title', start_date=derived_date, url=url, title=derived_title)
+            if df.empty:
+                return "", "", "", {"error": "No data found"}
+            processed_text = df['discussion'].iloc[0]
+
+        else:
+            if lang == 'es' and platform == 'wikipedia':
+                df = collect(mode='title', start_date=derived_date, url=url, title=derived_title, platform=platform, lang=lang, date='')
+            elif lang == 'gr' and platform == 'wikipedia':
+                if not date or len(date.split('/')) != 2:
+                    return "", "", "", {"error": "For Greek Wikipedia (title mode), please provide 'date' in 'mm/yyyy' format."}
+                df = collect(mode='title', start_date=derived_date, url=url, title=derived_title, platform=platform, lang=lang, date=date, years=[date])
+
+            elif lang == 'en' and platform == 'wikidata_entity':
+                df = collect(mode='url', url=url, platform=platform, lang=lang)
+            
+            elif lang == 'en' and platform == 'wikidata_property':
+                df = collect(mode='url', start_date=derived_date, url=url, title=derived_title, platform=platform, lang=lang, date=date)
+
+            elif lang == 'en' and platform == 'wikinews':
+                df = collect(mode='url', start_date=derived_date, url=url, title=derived_title, platform=platform, lang=lang, date=date)
+                
+            elif lang == 'en' and platform == 'wikiquote':
+                df = collect(mode='title', start_date=derived_date, url=url, title=derived_title, platform=platform, lang=lang, date=date)
+                
+            else:
+                return "", "", "", {"error": f"No implementation detail for lang={lang} platform={platform} provided."}
+
+            if isinstance(df, str) or df.empty:
+                return "", "", "", {"error": "No data returned from collect function."}
+
+            processed_text = df['discussion'].iloc[0]
+
+    else:
+        if lang == 'en' and platform == 'wikipedia':
+            df = prepare_dataset(mode='title', start_date=derived_date, url=url, title=derived_title)
+            if df.empty:
+                return "", "", "", {"error": "No data found"}
+            processed_text = df['discussion'].iloc[0]
+        else:
+            return "", "", "", {"error": f"Currently only English Wikipedia supported for {task_name}."}
+
+    final_scores = predict_text(processed_text, model_name, lang=lang, platform=platform, date=date)
+
     if model_name == 'outcome':
         highest_prob_item = max(final_scores, key=lambda x: x['score'])
         highest_prob_label = highest_prob_item['outcome']
         highest_prob = highest_prob_item['score']
         progress_bars = {item['outcome']: item['score'] for item in final_scores}
         
-        return processed_text, highest_prob_label, highest_prob, progress_bars
-    
+        return processed_text, highest_prob_label, str(highest_prob), progress_bars
     else:
         return processed_text, "", "", final_scores
 
-title = 'Wide-Analysis: A Wikipedia Deletion Discussion Analysis Suite'
-desc = """ Wide-Analysis is a suite of tools for analyzing Wikipedia deletion discussions. It is designed to help researchers and practitioners to understand the dynamics of deletion discussions, and to develop tools for supporting the decision-making process in Wikipedia. The suite includes a set of tools for collecting, processing, and analyzing deletion discussions. The package contains the following functionalities
-   
-     - Outcome Prediction: Predicting the outcome of a deletion discussion, the outcome can be the decision made with the discussion (e.g., keep, delete, merge, etc.) (determined from the complete discussion)
-     - Stance Detection: Identifying the stance of the participants in the discussion, in relation to the deletion decision.(determined from each individual comment in discussion)
-     - Policy Prediction: Predicting the policy that is most relevant to the comments of the participants in the discussion.(determined from each individual comment in discussion)
-     - Sentiment Prediction: Predicting the sentiment of the participants in the discussion, in relation to the deletion decision.(determined from each individual comment in discussion)
-     - Offensive Language Detection: Detecting offensive language in the comments of the participants in the discussion.(determined from each individual comment in discussion)
-
-The input to the classifier is a URL of a Wikipedia deletion discussion page with the task listed in the drop-down box, and the output is the predicted label of the discussion, along with the probability of the predicted label, and the probabilities of all the labels.
-
-"""
 
 url_input = gr.Textbox(label="URL")
 model_name_input = gr.Dropdown(label="Choose the Task", choices=list(model_dict.keys()), value=list(model_dict.keys())[0])
+lang_input = gr.Dropdown(label="Language", choices=lang_dict, value="en")
+platform_input = gr.Dropdown(label="Platform", choices=platform_dict, value="wikipedia")
+date_input = gr.Textbox(label="Date (if required)", placeholder="For ES year mode: dd/mm/yyyy, for GR title mode: mm/yyyy")
+
 outputs = [
     gr.Textbox(label="Processed Text"),
-    gr.Textbox(label="Label with Highest Probability"),  # This will only be used for the outcome task
-    gr.Textbox(label="Probability"),  # This will only be used for the outcome task
-    gr.JSON(label="All Labels and Probabilities")  # This will be used for all tasks
+    gr.Textbox(label="Label with Highest Probability"),  # Only for outcome
+    gr.Textbox(label="Probability"),  # Only for outcome
+    gr.JSON(label="All Labels and Probabilities")  # For all tasks
 ]
 
-demo = gr.Interface(fn=process_url, inputs=[url_input, model_name_input], outputs=outputs, title=title, description=desc)
-demo.launch()  # share=True
+demo = gr.Interface(fn=process_url, 
+                    inputs=[url_input, model_name_input, lang_input, platform_input, date_input],
+                    outputs=outputs, 
+                    title=title, 
+                    description=desc)
+demo.launch(share=True)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# import data_prep
+# import model_predict
+# import gradio as gr
+
+# # model_dict = {
+# #     "BERT-Base": "research-dump/bert-base-uncased_deletion_multiclass_complete_Final",
+# #     "BERT-Large": "research-dump/bert-large-uncased_deletion_multiclass_complete_final",
+# #     "RoBERTa-Base": "research-dump/roberta-base_deletion_multiclass_complete_final",
+# #     "RoBERTa-Large": "research-dump/roberta-large_deletion_multiclass_complete_final"
+# # }
+
+# model_dict = {
+#     "Outcome Prediction": "outcome",
+#     "Stance Detection": "stance",
+#     "Policy Prediction": "policy",
+#     "Sentiment Analysis": "sentiment",
+#     "Offensive Language Detection": "offensive"
+# }
+
+# def process_url(url, model_name):
+#     model_name = model_dict[model_name]
+#     processed_text = data_prep.process_data(url)
+#     final_scores = model_predict.predict_text(processed_text, model_name)
+#     if model_name == 'outcome':
+#         highest_prob_item = max(final_scores, key=lambda x: x['score'])
+#         highest_prob_label = highest_prob_item['outcome']
+#         highest_prob = highest_prob_item['score']
+#         progress_bars = {item['outcome']: item['score'] for item in final_scores}
+        
+#         return processed_text, highest_prob_label, highest_prob, progress_bars
+    
+#     else:
+#         return processed_text, "", "", final_scores
+
+# title = 'Wide-Analysis: A Wikipedia Deletion Discussion Analysis Suite'
+# desc = """ Wide-Analysis is a suite of tools for analyzing Wikipedia deletion discussions. It is designed to help researchers and practitioners to understand the dynamics of deletion discussions, and to develop tools for supporting the decision-making process in Wikipedia. The suite includes a set of tools for collecting, processing, and analyzing deletion discussions. The package contains the following functionalities
+   
+#      - Outcome Prediction: Predicting the outcome of a deletion discussion, the outcome can be the decision made with the discussion (e.g., keep, delete, merge, etc.) (determined from the complete discussion)
+#      - Stance Detection: Identifying the stance of the participants in the discussion, in relation to the deletion decision.(determined from each individual comment in discussion)
+#      - Policy Prediction: Predicting the policy that is most relevant to the comments of the participants in the discussion.(determined from each individual comment in discussion)
+#      - Sentiment Prediction: Predicting the sentiment of the participants in the discussion, in relation to the deletion decision.(determined from each individual comment in discussion)
+#      - Offensive Language Detection: Detecting offensive language in the comments of the participants in the discussion.(determined from each individual comment in discussion)
+
+# The input to the classifier is a URL of a Wikipedia deletion discussion page with the task listed in the drop-down box, and the output is the predicted label of the discussion, along with the probability of the predicted label, and the probabilities of all the labels.
+
+# """
+
+# url_input = gr.Textbox(label="URL")
+# model_name_input = gr.Dropdown(label="Choose the Task", choices=list(model_dict.keys()), value=list(model_dict.keys())[0])
+# outputs = [
+#     gr.Textbox(label="Processed Text"),
+#     gr.Textbox(label="Label with Highest Probability"),  # This will only be used for the outcome task
+#     gr.Textbox(label="Probability"),  # This will only be used for the outcome task
+#     gr.JSON(label="All Labels and Probabilities")  # This will be used for all tasks
+# ]
+
+# demo = gr.Interface(fn=process_url, inputs=[url_input, model_name_input], outputs=outputs, title=title, description=desc)
+# demo.launch()  # share=True