Added sentiment intensity plot

app/__init__.py

@@ -8,6 +8,11 @@ try:
 except LookupError:
     nltk.download("punkt")
 
+try:
+    nltk.data.find("sentiment/vader_lexicon.zip")
+except LookupError:
+    nltk.download("vader_lexicon")
+
 PATH_TO_COSMIC = "./Model/COSMIC"
 EXTRACTORS_PATH = PATH_TO_COSMIC + "/feature_extraction"
 EPIK_MODEL_DIR = PATH_TO_COSMIC + "/erc_training"

app/cosmic_view.py

@@ -11,6 +11,7 @@ from app.utils import (
     decode_speaker_role,
     display_sentiment_score_table,
     sentiment_flow_plot,
+    sentiment_intensity_analysis,
     EXAMPLE_CONVERSATIONS,
 )
 from fairseq.data.data_utils import collate_tokens
@@ -237,10 +238,45 @@ def cosmic_ui():
         plot_btn = gr.Button(value="Plot Sentiment Flow")
         plot_btn.click(sentiment_flow_plot, inputs=[output], outputs=[plot_box])
 
+        gr.Markdown("# Sentiment Intensity Analysis")
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown(
+                    """
+                        How accurate is the model? How good are the labels? These are
+                        some questions that we may have at this point, and we need to
+                        look at different metrics to assess the performance of our
+                        models. One of them is sentiment intensity which measures how
+                        strong a sentiment is expressed in the text. 
+                        
+                        This can be done by using NLTK's `SentimentIntensityAnalyzer`
+                        which analyzes the connotation of the words in the text and
+                        suggests whether a text is positive (with score > 0) or negative
+                        (score < 0) and at what degree the text is positive or negative.
+                        The graph to the right illustrates the change in sentiment
+                        intensity of the agent and visitor across the course of the
+                        conversation.
+                        
+                        <b><u>Note:</u></b> While NLTK's SentimentIntensityAnalyzer
+                        offers valuable insights, it is primarily trained on social media
+                        data like Twitter. Its performance might falter for lengthy or
+                        intricate messages. However, it remains a useful tool for
+                        gaining perspective on sentiment in conversations.
+                    """
+                )
+            with gr.Column(scale=2):
+                intensity_plot = gr.LinePlot()
+        intensity_plot_btn = gr.Button(value="Plot Sentiment Intensity")
+        intensity_plot_btn.click(
+            sentiment_intensity_analysis,
+            inputs=[conversation_input],
+            outputs=[intensity_plot],
+        )
+
         # reset all outputs whenever a change in the input is detected
         conversation_input.change(
-            lambda x: ("", None),
+            lambda x: ("", None, None),
             conversation_input,
-            outputs=[output, plot_box],
+            outputs=[output, plot_box, intensity_plot],
         )
     return cosmic_model

app/deberta_view.py

@@ -4,6 +4,7 @@ from app.utils import (
     format_prediction_ouptut,
     display_sentiment_score_table,
     sentiment_flow_plot,
+    sentiment_intensity_analysis,
     EXAMPLE_CONVERSATIONS,
 )
 
@@ -44,7 +45,7 @@ def deberta_ui():
     with gr.Blocks() as deberta_model:
         gr.Markdown(
             """
-            # Deberta
+            # DeBERTa
             Building upon the DeBERTa architecture, the model was customized and
             retrained on Epik data to classify messages between Visitors and Agents into
             corresponding sentiment labels. At the time of training by the team prior to
@@ -115,10 +116,45 @@ def deberta_ui():
         plot_btn = gr.Button(value="Plot Sentiment Flow")
         plot_btn.click(sentiment_flow_plot, inputs=[output], outputs=[plot_box])
 
+        gr.Markdown("# Sentiment Intensity Analysis")
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown(
+                    """
+                        How accurate is the model? How good are the labels? These are
+                        some questions that we may have at this point, and we need to
+                        look at different metrics to assess the performance of our
+                        models. One of them is sentiment intensity which measures how
+                        strong a sentiment is expressed in the text. 
+                        
+                        This can be done by using NLTK's `SentimentIntensityAnalyzer`
+                        which analyzes the connotation of the words in the text and
+                        suggests whether a text is positive (with score > 0) or negative
+                        (score < 0) and at what degree the text is positive or negative.
+                        The graph to the right illustrates the change in sentiment
+                        intensity of the agent and visitor across the course of the
+                        conversation.
+                        
+                        <b><u>Note:</u></b> While NLTK's SentimentIntensityAnalyzer
+                        offers valuable insights, it is primarily trained on social media
+                        data like Twitter. Its performance might falter for lengthy or
+                        intricate messages. However, it remains a useful tool for
+                        gaining perspective on sentiment in conversations.
+                    """
+                )
+            with gr.Column(scale=2):
+                intensity_plot = gr.LinePlot()
+        intensity_plot_btn = gr.Button(value="Plot Sentiment Intensity")
+        intensity_plot_btn.click(
+            sentiment_intensity_analysis,
+            inputs=[conversation_input],
+            outputs=[intensity_plot],
+        )
+
         # reset all outputs whenever a change in the input is detected
         conversation_input.change(
-            lambda x: ("", None),
+            lambda x: ("", None, None),
             conversation_input,
-            outputs=[output, plot_box],
+            outputs=[output, plot_box, intensity_plot],
         )
     return deberta_model

app/gpt_view.py

@@ -9,6 +9,7 @@ from app.utils import (
     create_input_instruction,
     sentiment_flow_plot,
     display_sentiment_score_table,
+    sentiment_intensity_analysis,
     EXAMPLE_CONVERSATIONS,
 )
 
@@ -339,14 +340,48 @@ def gpt_ui():
                 plot_box = gr.Plot(label="Analysis Plot")
 
         plot_btn = gr.Button(value="Plot Sentiment Flow")
-
         plot_btn.click(sentiment_flow_plot, inputs=[output_box], outputs=[plot_box])
 
+        gr.Markdown("# Sentiment Intensity Analysis")
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown(
+                    """
+                        How accurate is the model? How good are the labels? These are
+                        some questions that we may have at this point, and we need to
+                        look at different metrics to assess the performance of our
+                        models. One of them is sentiment intensity which measures how
+                        strong a sentiment is expressed in the text. 
+                        
+                        This can be done by using NLTK's `SentimentIntensityAnalyzer`
+                        which analyzes the connotation of the words in the text and
+                        suggests whether a text is positive (with score > 0) or negative
+                        (score < 0) and at what degree the text is positive or negative.
+                        The graph to the right illustrates the change in sentiment
+                        intensity of the agent and visitor across the course of the
+                        conversation.
+                        
+                        <b><u>Note:</u></b> While NLTK's SentimentIntensityAnalyzer
+                        offers valuable insights, it is primarily trained on social media
+                        data like Twitter. Its performance might falter for lengthy or
+                        intricate messages. However, it remains a useful tool for
+                        gaining perspective on sentiment in conversations.
+                    """
+                )
+            with gr.Column(scale=2):
+                intensity_plot = gr.LinePlot()
+        intensity_plot_btn = gr.Button(value="Plot Sentiment Intensity")
+        intensity_plot_btn.click(
+            sentiment_intensity_analysis,
+            inputs=[conversation_input],
+            outputs=[intensity_plot],
+        )
+
         # reset all outputs whenever a change in the input is detected
         conversation_input.change(
-            lambda x: ("", "", None),
+            lambda x: ("", "", None, None),
             conversation_input,
-            outputs=[output_box, report_md, plot_box],
+            outputs=[output_box, report_md, plot_box, intensity_plot],
         )
 
     return gpt_model

app/utils.py

@@ -1,8 +1,13 @@
-import gradio as gr
 import os
 import re
+import nltk
+import gradio as gr
+import pandas as pd
 import seaborn as sns
+from statistics import mean
 import matplotlib.pyplot as plt
+from nltk.sentiment import SentimentIntensityAnalyzer
+from preprocessing.preprocess import process_user_input
 from preprocessing.preprocess import decode_numeric_label, decode_speaker_role
 
 
@@ -107,7 +112,7 @@ def sentiment_flow_plot(labeled_conv):
     fig, ax = plt.subplots()
     sns.set(style="whitegrid")
 
-    ax.plot(X, visitor_Y_converted, label="Visitor", color="blue", marker="o")
+    ax.plot(X, visitor_Y_converted, label="Visitor", color="red", marker="o")
     ax.plot(X, agent_Y_converted, label="Agent", color="green", marker="o")
 
     plt.legend(loc="upper left", bbox_to_anchor=(1, 1))
@@ -123,6 +128,71 @@ def sentiment_flow_plot(labeled_conv):
     return fig
 
 
+def sentiment_intensity_analysis(input):
+    def analyze_sentiment(text):
+        sia = SentimentIntensityAnalyzer()
+        scores = [
+            sia.polarity_scores(sentence)["compound"]
+            for sentence in nltk.sent_tokenize(text)
+        ]
+        return mean(scores)
+
+    result = process_user_input(input)
+
+    if not result["success"]:
+        raise gr.Error(result["message"])
+
+    data = result["data"]
+    speakers = [item[1] for item in data]
+    messages = [item[2] for item in data]
+
+    df = pd.DataFrame(columns=["Speaker", "Sentiment Intensity", "Timestamp"])
+
+    visitor_previous = {
+        "Speaker": ["Visitor"],
+        "Sentiment Intensity": [0],
+        "Timestamp": [0],
+    }
+    agent_previous = {
+        "Speaker": ["Agent"],
+        "Sentiment Intensity": [0],
+        "Timestamp": [0],
+    }
+    for i in range(len(messages)):
+        new_row = {
+            "Speaker": [speakers[i]],
+            "Sentiment Intensity": [analyze_sentiment(messages[i])],
+            "Timestamp": [i + 1],
+        }
+        df = pd.concat([df, pd.DataFrame(new_row)], ignore_index=True)
+
+        # Duplicate the sentiment of the person that does not speak at this duration
+        if speakers[i] == "Agent":
+            # Agent speak this turn, duplicate the sentiment of the visitor
+            visitor_previous["Timestamp"] = [i + 1]
+            df = pd.concat([df, pd.DataFrame(visitor_previous)], ignore_index=True)
+            agent_previous = new_row
+        else:
+            # Visitor speak, duplicate the sentiment of the agent
+            agent_previous["Timestamp"] = [i + 1]
+            df = pd.concat([df, pd.DataFrame(agent_previous)], ignore_index=True)
+            visitor_previous = new_row
+
+    return gr.LinePlot(
+        df,
+        label="Intensity Plot",
+        x="Timestamp",
+        y="Sentiment Intensity",
+        color="Speaker",
+        color_legend_position="right",
+        title="Conversation Sentiment Intensity",
+        tooltip=["Speaker", "Sentiment Intensity"],
+        y_lim=[-1, 1],
+        height=400,
+        width=400,
+    )
+
+
 SENTIMENT_SCORE_TABLE = """
 <div align="center" style='font-size: small;'>
 
@@ -149,6 +219,6 @@ def display_sentiment_score_table():
 
 EXAMPLE_CONVERSATIONS = {
     "Example 1": "Visitor: Hey Jess, available for outcall tonight?\n\nAgent: hey hey what you looking for luv?\n\nVisitor: Some company in El Segundo\n\nAgent: out with friends or just the two of us?\n\nVisitor: Just the two\n\nAgent: looking for some connection then...\n\nVisitor: Yes\n\nAgent: I get it. Looking for companionship, it's a very legit need. I'll level with you. We're a group of guys trying to show other men the harms of the sex trade industry, both to buyers and sellers. You up for a little conversation? No pressure... no judgement.\n\nVisitor: No, but interested in the harms in companionship?\n\nAgent: The truth is nearly all of the women on these sites are not in this career by choice. We work with survivors. Their money goes to pimps.. their lives are controlled.\n\nVisitor: Really?\n\nVisitor: Even when they say \"independent\"?\n\nAgent: Yes. It's all part of the human trafficking system... pimps find & recruit nearly all women who are doing this, they are very, very good at manipulation and control, and working in the background. Sadly, this modern day slavery will be with us as long as there are willing buyers. That's why we are trying to work this angle. The more men that know the truth, the fewer will want to exploit someone in this way.\n\nAgent: I need to move on, but just consider what I've told you tonight. Polarisproject.org is a great resource to learn more... it's nothing like what Hollywood shows ya. We work with the women & some guys who have escaped out of this lifestyle. It is modern-day slavery, straight up. Thanks for the chat. Good night!\n\nVisitor: Thanks, fyi it worked. Hung it up, I'm to old to me messin around in this crazy stuff take care\n\nAgent: I am so glad to hear this friend. You calling this number tonight was not just a coincidence. Take care also!!\n\n",
-    "Example 2": "Visitor: What's up sexy\n\nAgent: Thank you! I'm hangin\n\nVisitor: Are you available at the moment\n\nAgent: what's your name\n\nVisitor: David ... Yours\n\nAgent: Dakota is not available right now, but my name is Jeff and I'm working with a bunch of guys trying to curb the demand for victims of human trafficking. Did you ever consider that Dakota could be a victim of trafficking?\n\nVisitor: Fuck no I just wanted to get some pussy .... Bye\n\nAgent: Just so you know your number is in our system now. Understand that next time you click on one of these ads you could be walking into an undercover cop and many of these girls are underage. That is a crime that will follow you the rest of your life!\n\nVisitor: . You think the predators your looking for going to fall into your lap? Lol your not a cop stop trying to be one ... You want to help?\n\nVisitor: Just so you know I don't give no fucks about you or no fucking cops... Threatening me will not get you shit from me ,no info no nothing you stupid cunt..\n\nAgent: Hey man, I did not want to threaten you. I want you to be aware of the risks. Have you heard that most women in prostitution do it against their will, and many are minors?\n\nVisitor: I never fuck with minors that's for sure cause you can tell if they are not of age\n\nAgent: Yeah, many people think it's harmless and these women are in complete control. But, the truth is, most of these women are enslaved, forced to sell themselves by pimps who take 100% of their money. I hope you can see these women in a different light.\n\nVisitor: I know most are blacks living off a woman cause they can't get no job ... I won't see a female if she has a guy around\n\nAgent: I'm curious, is there something missing inside of you, or are you angry about something not right in your life that causes you to seek out connection with these girls?\n\nVisitor: No actually I am on gps monitor and can't go out and meet no one so I got needs and feel sometimes hooking up with one of these girls here and there is w\n\nVisitor: hat works for my situation....but I'm not disrespectful to them or be doing ugly things ... My ex wife cheated on me after 8yrs of marriage I don't reall\n\nVisitor: y trust women like enough for a relationship\n\nVisitor: I'm being honest with you cause even tho I don't know u I believe in doing the right thing and your movement with alot of work is going to save lives so\n\nVisitor: that's good\n\nAgent: Hey David, I'm very sorry for your situation and the pain you must be in. Were you raised with any faith perspective?\n\nVisitor: Yes I actually was baptized for the first time in my life in 2018 and have made some drastic changes to my life.... I'm not perfect but I am working hard\n\nAgent: Okay, David. That's great news. We're all a work in progress. God is faithful to forgive us of our sins if we just ask him. Read 1 John 1:9. Can I direct you to some resources to help you overcome this urge to buy sex?\n\nVisitor: Honestly I'm not ready for that is all I can say at the moment.... Sorry but I don't like to lie\n\nAgent: Hey, no worries. When you're ready, check out www.celebraterecovery.com as they have a lot of resources that might help. I will be praying for you, David. I've had a lot of pain in my life and only in my relationship with Jesus have I found true peace that surpasses my understanding. I pray that for you too.\n\nVisitor: What kinda pain ? Do you mind sharing?\n\nVisitor: And what are y'all doing to put those pieces of shit in prison who are hurting these women\n\nAgent: I've had several people close to me pass away tragically in the past few years. Thanks for asking. I'm just trusting God each day trusting he's in control.\n\nAgent: Hey, honestly, the best way we can solve this problem of evil pieces of shit harming these girls is to dry up the demand for these girls by encouraging men like us to stop calling these ads. If the demand dries up, evil predatory scums' business with die.\n\nAgent: Can I get your commitment to do your part and stop engaging these girls on their escort ads?\n\nVisitor: Yes I will but I can promise you that it will not stop matter of fact it will get worse ... They will beat them more and make them do it more only way is\n\nVisitor: to put the pimps in prison for alot of years....\n\nAgent: Hey, I'll be praying for you. Feel free to call me any time if you need to talk. My number is (615) 628-7545. God bless you, brother.\n\nVisitor: Thank u\n\n",
+    "Example 2": "Visitor: What's up sexy\n\nAgent: Thank you! I'm hangin\n\nVisitor: Are you available at the moment\n\nAgent: what's your name\n\nVisitor: David ... Yours\n\nAgent: Dakota is not available right now, but my name is Jeff and I'm working with a bunch of guys trying to curb the demand for victims of human trafficking. Did you ever consider that Dakota could be a victim of trafficking?\n\nVisitor: Fuck no I just wanted to get some pussy .... Bye\n\nAgent: Just so you know your number is in our system now. Understand that next time you click on one of these ads you could be walking into an undercover cop and many of these girls are underage. That is a crime that will follow you the rest of your life!\n\nVisitor: . You think the predators your looking for going to fall into your lap? Lol your not a cop stop trying to be one ... You want to help?\n\nVisitor: Just so you know I don't give no fucks about you or no fucking cops... Threatening me will not get you shit from me ,no info no nothing you stupid cunt..\n\nAgent: Hey man, I did not want to threaten you. I want you to be aware of the risks. Have you heard that most women in prostitution do it against their will, and many are minors?\n\nVisitor: I never fuck with minors that's for sure cause you can tell if they are not of age\n\nAgent: Yeah, many people think it's harmless and these women are in complete control. But, the truth is, most of these women are enslaved, forced to sell themselves by pimps who take 100% of their money. I hope you can see these women in a different light.\n\nVisitor: I know most are blacks living off a woman cause they can't get no job ... I won't see a female if she has a guy around\n\nAgent: I'm curious, is there something missing inside of you, or are you angry about something not right in your life that causes you to seek out connection with these girls?\n\nVisitor: No actually I am on gps monitor and can't go out and meet no one so I got needs and feel sometimes hooking up with one of these girls here and there is what works for my situation....but I'm not disrespectful to them or be doing ugly things ... My ex wife cheated on me after 8yrs of marriage I don't really trust women like enough for a relationship\n\nVisitor: I'm being honest with you cause even tho I don't know u I believe in doing the right thing and your movement with alot of work is going to save lives so\n\nVisitor: that's good\n\nAgent: Hey David, I'm very sorry for your situation and the pain you must be in. Were you raised with any faith perspective?\n\nVisitor: Yes I actually was baptized for the first time in my life in 2018 and have made some drastic changes to my life.... I'm not perfect but I am working hard\n\nAgent: Okay, David. That's great news. We're all a work in progress. God is faithful to forgive us of our sins if we just ask him. Read 1 John 1:9. Can I direct you to some resources to help you overcome this urge to buy sex?\n\nVisitor: Honestly I'm not ready for that is all I can say at the moment.... Sorry but I don't like to lie\n\nAgent: Hey, no worries. When you're ready, check out www.celebraterecovery.com as they have a lot of resources that might help. I will be praying for you, David. I've had a lot of pain in my life and only in my relationship with Jesus have I found true peace that surpasses my understanding. I pray that for you too.\n\nVisitor: What kinda pain ? Do you mind sharing?\n\nVisitor: And what are y'all doing to put those pieces of shit in prison who are hurting these women\n\nAgent: I've had several people close to me pass away tragically in the past few years. Thanks for asking. I'm just trusting God each day trusting he's in control.\n\nAgent: Hey, honestly, the best way we can solve this problem of evil pieces of shit harming these girls is to dry up the demand for these girls by encouraging men like us to stop calling these ads. If the demand dries up, evil predatory scums' business with die.\n\nAgent: Can I get your commitment to do your part and stop engaging these girls on their escort ads?\n\nVisitor: Yes I will but I can promise you that it will not stop matter of fact it will get worse ... They will beat them more and make them do it more only way is\n\nVisitor: to put the pimps in prison for alot of years....\n\nAgent: Hey, I'll be praying for you. Feel free to call me any time if you need to talk. My number is (615) 628-7545. God bless you, brother.\n\nVisitor: Thank u\n\n",
     "Example 3": "Visitor: Heyyy\n\nVisitor: How are you this evening\n\nAgent: better now ;) call me\n\nVisitor: Iam at work for now ,be off around 10pm\n\nVisitor: Need some company\n\nVisitor: Are you independent honey\n\nAgent: well since you arent available at the moment ill just come out and say-these sites are bad news. did you know that most of the girls on here are here against their will? Most of them got dragged into this lifestyle by an abuser, oftentimes before they were of legal consenting age. isnt that sad?\n\nAgent: we are with some guys who are trying to spread awareness of the realities of this \"industry\".\n\nAgent: https://exoduscry.com/choice/\n\nVisitor: Thanks\n\nAgent: i encourage you to watch this video. it is jarring to think about how bad someone else's options must be to choose to be on these sites\n\nVisitor: Ooohhh\n\nAgent: selling their body to make ends meet or appease a pimp\n\nVisitor: That's really awful\n\nAgent: it is. you seem like the kind of guy who wouldnt wont to proliferate that kind of harmful lifestyle. am i right in thinking that?\n\nVisitor: Well iam just looking for attention\n\nVisitor: My marriage is not going well lol\n\nAgent: i know that it is hard to find ourselves lonely and without much alternative to meet that perceived need but its humbling to think that our needs can force someone else into such a dark place\n\nAgent: hey, thanks for sharing that my man. i know it can be hard\n\nAgent: marraige is the most humbling of relationships, isnt it?\n\nVisitor: She leaves with her friends n no time for me\n\nAgent: ive been there my guy. i know that it is alot easier to numb that loneliness for sure\n\nVisitor: I want to be faithful\n\nAgent: does your wife know how you feel when she chooses her friends instead of you?\n\nVisitor: I been drinking lately\n\nVisitor: Yes , she takes pills\n\nAgent: if so, i hope you are praying for her to realize the hurt she is causing and to seek change\n\nVisitor: She had surgery 4 yes ago n it's been hard for her n her addiction on pills\n\nVisitor: Yes for now iam looking for a female friend to talk n see what can we do for each other\n\nAgent: that is hard my man. physical pain is a huge obstacle in life for sure so i hear you\n\nVisitor: Well chat later .thanks\n\nAgent: have you considered pursuing other men who can encourage you instead of looking for the easy way out?\n\nAgent: what is your name my friend? i will be praying for you by name if you wouldnt mind sharing it\n\nAgent: well, i gotta run. watch that video i sent and i will definitely be praying for you. I hope you pray for yourself and for your wife - God can definitely intervene and cause complete change in the situation if He wills it. He is good and He hears you. You are loved by Him, brother. Good night\n\n",
 }