MVP

README copy.md

@@ -1,12 +0,0 @@
----
-title: SalesBuddy for BetterTech
-emoji: 👁
-colorFrom: pink
-colorTo: blue
-sdk: docker
-pinned: false
-license: mit
-short_description: BetterTech Lending Analytics provides a comprehensive solution for real-time analysis in lending, surpassing basic regulatory compliance by enabling reporting, explaining, and optimizing. Deployed as SaaS, on-premise, or in the cloud, it empowers organizations to improve performance, enhance decision-making, and stay competitive by simulating various scenarios efficiently and accurately.
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README.md

@@ -1,11 +1,12 @@
 ---
-title: SalesBuddy
-emoji: ⚡
+title: SalesBuddy for BetterTech
+emoji: 👁
 colorFrom: pink
-colorTo: indigo
+colorTo: blue
 sdk: docker
 pinned: false
 license: mit
+short_description: BetterTech Lending Analytics provides a comprehensive solution for real-time analysis in lending, surpassing basic regulatory compliance by enabling reporting, explaining, and optimizing. Deployed as SaaS, on-premise, or in the cloud, it empowers organizations to improve performance, enhance decision-making, and stay competitive by simulating various scenarios efficiently and accurately.
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,30 +1,31 @@
+import asyncio
 import chainlit as cl
+import json
 import os
+from classes import SessionState
+from datetime import datetime
 from dotenv import load_dotenv
-import asyncio
-from langchain_core.runnables.config import RunnableConfig
 from langchain_openai import ChatOpenAI
-from datetime import datetime, timedelta
-import pyttsx3
-from tabulate import tabulate
 from utils_data import  get_company_data, get_opportunities, get_questions, get_customer_background
-import json
 from utils_prompt import get_user_template, get_system_template
-from utils_evaluate import get_evaluation   
 from utils_prompt import get_chat_prompt
-from classes import SessionState
-from utils_output import create_html_dashboard
-import threading
-import dashboard_server
+from utils_output import display_evaluation_results, display_llm_responses
 
-server_thread = threading.Thread(target=dashboard_server.run_server, daemon=True)
-server_thread.start()
-
-DASHBOARD_SERVER_URL = f"http://localhost:{dashboard_server.PORT}"
+llm_model = "gpt-4o-mini"
+# llm_model = "gpt-4o"
+# llm_model = "gpt-3.5-turbo"
+# llm_model = "gpt-4o-2024-08-06"
 
 load_dotenv()
 openai_api_key = os.getenv("OPENAI_API_KEY")
 
+user_template = get_user_template()
+system_template = get_system_template()
+
+#############################################
+# Action callbacks
+#############################################
+
 @cl.action_callback("Run a Scenario")
 async def on_action(action):
     scenarios = get_opportunities()
@@ -45,29 +46,26 @@ async def on_action(action):
                             f"({row['Customer Contact Role']})"
             )
             scenario_actions.append(scenario_action)
-
-    # Send message with action buttons for the user to choose
     await cl.Message(content="Select a scenario (hover for details):", actions=scenario_actions).send()
 
 @cl.action_callback("Scenario")
 async def on_action(action):
-    await cl.Message(content="Please wait, I am gathering the scenario information...").send()
+    await cl.Message(content="Please wait, I am gathering information...").send()
     index = int(action.value)
     scenarios = cl.user_session.get("scenarios", None)
     if scenarios is None:
         await cl.Message(content="No scenarios found.").send()
         return
     await cl.Message(content="...gathering scenario information").send()
-    # await asyncio.sleep(1) 
+    await asyncio.sleep(1) 
 
     await cl.Message(content="...creating questions").send()  
-    # await asyncio.sleep(3)
-    # get scenario, customer, company
+    await asyncio.sleep(1)
+
     selected_scenario = scenarios.iloc[index]
     this_session = cl.user_session.get("session", None)
     this_session.add_scenario_info(selected_scenario) 
     get_customer_background(this_session, selected_scenario['Customer Name'])
-    # get questions
     this_session.questions = get_questions(this_session.opportunity.stage, this_session.num_questions)
 
     opening_message = this_session.get_opening()  
@@ -82,38 +80,25 @@ async def on_action(action):
     this_session = cl.user_session.get("session", None)
     start_time = datetime.now()
     this_session.start_time = start_time
-    await cl.Message(content=f"{this_session.customer.contact_name} joins the zoom call").send()
+    output = f"{this_session.customer.contact_name} joins the zoom call"
+    print(output)
+    await cl.Message(content=output).send()
 
-# @cl.action_callback("ask question")
-# async def on_action(action):
-#     print(questions_and_answers.columns)
-#     print(questions_and_answers.head())
-#     asked_question = cl.user_session.get("asked_question", 0)
-#     question = questions_and_answers.iloc[asked_question]["Customer Question (Provided by Claude)"]
-#     print(f"Question: {question}")
-#     print(question)
-#     asked_question += 1
-#     cl.user_session.set("asked_question", asked_question)
-#     cl.user_session.set("question", question)
-
-#     message = f"The question will be: {question}"
-#     await send_message(message)
-#     await cl.Message(message).send()
-
-@cl.action_callback("evaluate")
+@cl.action_callback("Evaluate Performance")
 async def on_action(action):
-    print("evaluation")
-    cl.user_session.set("evaluation", "true")
-    out_text = get_evaluation()
-    await cl.Message(content=out_text).send()
-
-user_template = get_user_template()
-system_template = get_system_template()
+    this_session = cl.user_session.get("session", None)
+    if this_session is None:
+        await cl.Message(content="No session found.").send()
+        return  
+    await display_evaluation_results(cl, this_session)
 
-def speak_text(text):
-    engine = pyttsx3.init(driverName='sapi5')
-    engine.say(text)
-    engine.runAndWait()
+@cl.action_callback("Display Queries and Responses")
+async def on_action(action):
+    this_session = cl.user_session.get("session", None)
+    if this_session is None:
+        await cl.Message(content="No session found.").send()
+        return  
+    await display_llm_responses(cl, this_session)   
 
 #############################################
 ### On Chat Start (Session Start) Section ###
@@ -122,14 +107,13 @@ def speak_text(text):
 async def on_chat_start():
     this_session = SessionState()
     get_company_data(this_session)
-    print(this_session)
     cl.user_session.set("session", this_session)
 
     cl.user_session.set("message_count", 0)
     cl.user_session.set("follow_up_questions", 0)
 
     chat_prompt = get_chat_prompt()
-    chat_model = ChatOpenAI(model="gpt-4o-mini")
+    chat_model = ChatOpenAI(model=llm_model)
     simple_chain = chat_prompt | chat_model
     cl.user_session.set("chain", simple_chain)
 
@@ -137,49 +121,30 @@ async def on_chat_start():
     await cl.Message(content=welcome_message).send()
     await cl.Message(content=this_session.company.product_summary).send()
 
-    misc_actions = [
-        cl.Action(name="Run a Scenario", value="run-scenario", description="Run a scenario"),
-        cl.Action(name="Evaluate", value="evaluate", description="Evaluate")
-    ]
-    await cl.Message(content="Select an action:", actions=misc_actions).send()  
- 
-
-
-# async def send_message(question):
-#     history = cl.user_session.get("history", [])
-#     message_count = cl.user_session.get("message_count", 0)
-#     message_count += 1
-#     cl.user_session.set("message_count", message_count)
-#     chain = cl.user_session.get("chain")
-#     company = cl.user_session.get("company", "Acme Corp")
-#     role = cl.user_session.get("role", "ceo")
-#     attitude = cl.user_session.get("attitude", "happy")
-#     personality = cl.user_session.get("personality", "sarcastic")
-#     twist = cl.user_session.get("twist", "")
-#     answer = "123"
-#     follow_up_questions = cl.user_session.get("follow_up_questions", 0) 
-#     follow_up_questions += 1
-#     cl.user_session.set("follow_up_questions", follow_up_questions)
-
-#     if message_count == 2:
-#         twist = "The fire alarm is going off"
-#     msg = cl.Message(content="")
-
-#     history.append({"role": "user", "content": question})
-#     # handle streaming of LLM responses 
-#     response_content = ""
+    scenarios = get_opportunities()
+    cl.user_session.set("scenarios", scenarios)
+    scenarios = cl.user_session.get("scenarios", None)
+    if scenarios is None:
+        await cl.Message(content="No scenarios found.").send()
+        return
+    
+    scenario_actions = []
+    for idx, row in scenarios.iterrows():
+        if row['Opportunity Description'] != "":
+            scenario_action = cl.Action(
+                name="Scenario",
+                value=f"{idx}",  # Send the row index as value
+                description=f"{row['Customer Name']}: {row['Opportunity Name']} ({row['Opportunity Stage']}) "
+                            f"Value: {row['Opportunity Value']}. Meeting with {row['Customer Contact']} "
+                            f"({row['Customer Contact Role']})"
+            )
+            scenario_actions.append(scenario_action)
+    await cl.Message(content="Select a scenario (hover for details):", actions=scenario_actions).send()
 
-#     async for chunk in chain.astream(
-#         {"question": question, "company": company, "role": role, "attitude": attitude, "twist": twist, "conversation_history": history, "personality": personality, "answer": answer, "follow_up_questions": follow_up_questions },
-#         config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
-#     ):
-#         response_content += chunk.content
-#         await msg.stream_token(chunk.content)
-#     speak_text(response_content)
-#     history.append({"role": "assistant", "content": response_content})
-#     cl.user_session.set("history", history)
-#     await msg.send()
 
+#########################################################
+### On Message Section - called for each User Message ###
+#########################################################
 @cl.on_message
 async def main(message: cl.Message):
     content = message.content.strip()
@@ -191,99 +156,127 @@ async def main(message: cl.Message):
         
     else:
         if this_session.status == "active":
-
             chain = cl.user_session.get("chain")
-
             history = cl.user_session.get("history", [])
             history.append({"role": "user", "content": message})
             this_session.previous_answer = message.content
-
             prompt_parm = prepare_chain_parameters(this_session, message, history)
-            # handle streaming of LLM responses 
+            this_session.queries.append(prompt_parm)
             response_content = chain.invoke(prompt_parm)
             json_str = response_content.content.strip('```json\n').strip('```')
-            this_response = json.loads(json_str)
+            try:
+                this_response = json.loads(json_str)
+            except json.JSONDecodeError as e:
+                print(f"JSON Decode Error: {e}")
+                print(response_content.content)
+                print(f"Error at position {e.pos}: {json_str[max(0, e.pos-10):e.pos+10]}")
+                this_response = {"Response": "Error receiving response from LLM"}  
+            llm_response = this_response.get("Response", "No response from LLM")
+            print("LLM Response:")
+            print(llm_response)
+            this_session.llm_responses.append(this_response)
+            print("Next question:")
+            print(this_response.get("Question", "No question"))
+
             if this_session.question != "":
                 this_session.responses.append({
                     "question_number": this_session.current_question_index,
                     "question": this_session.question,
                     "response": this_session.rep_answer,
                     "ground_truth": this_session.ground_truth,
-                    "response_score": this_response.get("Score"),
-                    "response_evaluation": this_response.get("Evaluation"),
-                    "mood_score": this_response.get("Mood Score"),
-                    "overall_evaluation": this_response.get("Overall Evaluation"),
+                    "response_score": this_response.get("Score", "No score"),
+                    "response_evaluation": this_response.get("Evaluation", "No evaluation"),
+                    "mood_score": this_response.get("Mood Score", "No mood score"),
+                    "overall_score": this_response.get("Overall Score", "No overall score"),
+                    "overall_evaluation": this_response.get("Overall Evaluation", "No overall evaluation"),
                 })
-
-            await cl.Message(response_content.content).send()
+            message_to_rep = llm_response + "\n\n" + this_response.get("Question", "No question")
+            if this_session.do_voice:
+                print(f"Voice Response: {message_to_rep}")
+            else:
+                await cl.Message(message_to_rep).send()
+                # await cl.Message(this_response).send()
 
             history.append({"role": "assistant", "content": response_content})
             cl.user_session.set("history", history)
             this_session.current_question_index += 1
             if this_session.current_question_index > len(this_session.questions):
                 this_session.status = "complete"
-        else:
-            out_text = "All questions answered. Preparing evaluation results..."
-            await cl.Message(content=out_text).send()
-            await asyncio.sleep(1)
-            dashboard_path = create_html_dashboard(this_session.responses)
-            dashboard_filename = create_html_dashboard(this_session.responses)
-            
-            # Create a link to the dashboard
-            dashboard_link = f"{DASHBOARD_SERVER_URL}/{dashboard_filename}"
-            print(dashboard_link)
-            await cl.Message(content="View the latest Response Dashboard:").send()
-            # await cl.Link(url=dashboard_link, label="Response Dashboard", target="_blank").send()
-            await cl.Message(content=f"View the [Evaluation Dashboard]({dashboard_link})").send()
-
+                end_time = datetime.now()
+                duration = end_time - this_session.start_time
+                duration_minutes = round(duration.total_seconds() / 60)
+                this_session.end_time = end_time
+                this_session.duration_minutes = duration_minutes
+                
+                if this_session.do_evaluation:
+                    await display_evaluation_results(cl, this_session)
+                else:
+                    evaluate_actions = [
+                        cl.Action(name="Evaluate Performance", value="evaluate", description="Evaluate Performance"),
+                        cl.Action(name="Display Queries and Responses", value="display_llm_responses", description="Display LLM Responses")
+                    ]
+                    await cl.Message(content="Click to evaluate", actions=evaluate_actions).send()
 
-def display_responses(responses):
-    table_data = []
-    for resp in responses:
-        table_data.append([
-            resp["question_number"],
-            resp["question"],
-            resp["response"],
-            resp["ground_truth"],
-            resp["response_score"],
-            resp["response_evaluation"],
-            resp["mood_score"],
-            resp["overall_evaluation"]
-        ])
-    return table_data
+#############################################
+### Support Functions
+#############################################
+    
+# def display_responses(responses):
+#     table_data = []
+#     for resp in responses:
+#         table_data.append([
+#             resp["question_number"],
+#             resp["question"],
+#             resp["response"],
+#             resp["ground_truth"],
+#             resp["response_score"],
+#             resp["response_evaluation"],
+#             resp["mood_score"],
+#             resp["overall_evaluation"]
+#         ])
+#     return table_data
 
 def prepare_chain_parameters(this_session, message, history):
-    print("")
-    print("prepare_chain_parameters()")
-    print(message.content)
     message = message.content
     previous_question = ""
     rep_answer = ""
     next_question = ""  
     ground_truth = ""
+    command = ""
     if this_session.current_question_index == 0:
         previous_question = ""
         rep_answer = ""
         ground_truth = ""
         next_question = this_session.questions[this_session.current_question_index]["question"]
+        command = "You should greet the rep"
     elif this_session.current_question_index >= len(this_session.questions):
         next_question = "" 
         previous_question = this_session.questions[this_session.current_question_index - 1]["question"]
         rep_answer = this_session.previous_answer
         ground_truth = this_session.questions[this_session.current_question_index - 1]["ground_truth"]
+        command = """Thank the customer, offer a comment on the answer and overall performance.
+            Conclude the conversation with a summary and give a farewell. 
+            If the answers were good, give a positive farewell and offer a follow up meeting. 
+            If the answers were poor, give a poor farewell.
+            You can add additional comments as needed.
+        """
     else:
         previous_question = this_session.questions[this_session.current_question_index - 1]["question"]
         rep_answer = this_session.previous_answer
         next_question = this_session.questions[this_session.current_question_index]["question"]
         ground_truth = this_session.questions[this_session.current_question_index]["ground_truth"]
+        command = "You should respond to the answer based on how well the rep answered the previous question."
     this_session.ground_truth = ground_truth
     this_session.question = previous_question
     this_session.rep_answer = rep_answer
+    print("--------------------------------")
+    print(f"Message: {message}")
     print("Sending the following:")
+    print(f"Command: {command}")
     print(f"Previous question: {previous_question}")
     print(f"Rep answer: {rep_answer}")
     print(f"Next question: {next_question}")
-    print(f"Message: {message}")
+
     rep_company_details = f"""
         Name: {this_session.company.name}
         Description: {this_session.company.description}
@@ -317,6 +310,7 @@ def prepare_chain_parameters(this_session, message, history):
             "next_question": next_question, 
             "rep_answer": rep_answer,   
             "conversation_history": history,   
+            "command": command,
             }
     return parm
 
@@ -325,12 +319,9 @@ async def handle_control_message(command: str):
     main_command = command_parts[0].lower()
 
     if main_command == 'start':
-        # Handle start command
         await cl.Message(content="Starting new session...").send()
-        # Add your session start logic here
     
     elif main_command == 'stop':
-        # Handle end command
         this_session = cl.user_session.get("session")
         end_time = datetime.now()
         duration = end_time - this_session.start_time
@@ -339,14 +330,10 @@ async def handle_control_message(command: str):
         this_session.duration_minutes = duration_minutes
         
         await cl.Message(content=f"Ending current session after {this_session.duration_minutes} minutes").send()
-
-        # Add your session end logic here
     elif main_command == 'pause':
-        # Handle end command
         await cl.Message(content="Ending current session...").send()
 
     elif main_command == 'time':
-        # Handle time command (display current session duration)
         this_session = cl.user_session.get("session")
         duration = this_session.get_session_duration()
         await cl.Message(content=f"Current session duration: {duration}").send()

chainlit.md

@@ -1,2 +1,2 @@
-# Sideckick POC
+# Sales Buddy
 

classes.py

@@ -1,4 +1,6 @@
 class SessionState:
+    do_evaluation = False
+    do_voice = False
     status = "active"
     scenario = None
     qa_mode = "single"
@@ -8,13 +10,17 @@ class SessionState:
     duration_minutes = None
     attitude = "Happy"
     mood_score = 5
-    num_questions = 2
+    num_questions = 1
     current_question_index = 0
     previous_answer = None
     question = ""
     ground_truth = ""
     rep_answer = ""
     responses = []
+    queries = []
+    llm_responses = []
+    command = ""
+    scores = []
     class Company:
         def __init__(self, name, description, product, product_summary, product_description):
             self.name = name

dashboard_server.py

@@ -1,18 +0,0 @@
-import http.server
-import socketserver
-import os
-
-PORT = 8080
-DIRECTORY = "dashboards"
-
-class Handler(http.server.SimpleHTTPRequestHandler):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, directory=DIRECTORY, **kwargs)
-
-def run_server():
-    with socketserver.TCPServer(("", PORT), Handler) as httpd:
-        print(f"Serving dashboards at port {PORT}")
-        httpd.serve_forever()
-
-if __name__ == "__main__":
-    run_server()

dashboards/dashboard_04586f45c89443b3a7c26e295523cc70.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>0</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td></td>
-            <td></td>
-            <td>Our customers have to consider annual running costs (consumption), one time implementation fee, training costs, and support costs as they come on to our platform. Customers have cited productivity gains of 25% and analytics costs going down by 30% within 18 months post implementation.  </td>
-            <td>1</td>
-            <td>The response was not helpful or informative. It provided no relevant information regarding the total cost of ownership, which is crucial for my decision-making process.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_057256deb39742fcaa24d4f529fcf20f.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, thanks for reaching out! I'm interested to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, I appreciate your response, but it seems I didn't get the specific details I was looking for regarding the total cost of ownership for your solution. Could you provide me with a breakdown of that information?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response did not provide any useful information about the total cost of ownership. It was vague and unhelpful.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_1fe6095d0e794299814937ab8bf8fd81.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>0</td>
-            <td></td>
-            <td></td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>dsf</td>
-            <td>Our customers have to consider annual running costs (consumption), one time implementation fee, training costs, and support costs as they come on to our platform. Customers have cited productivity gains of 25% and analytics costs going down by 30% within 18 months post implementation.  </td>
-            <td>1</td>
-            <td>The response did not provide any relevant information regarding the total cost of ownership for the solution. Therefore, I cannot consider this a satisfactory answer.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_3a39eb7d2a124e8a98fb80e8c9e283e9.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, I'm doing well, thank you! I have a question: What's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Thank you for the information, Tony! It helps to have clarity on the cost.</td>
-            <td></td>
-            <td>5</td>
-            <td>The answer was straightforward and provided the requested information clearly. However, it lacks additional context or breakdown regarding what the cost includes, which could have been beneficial.</td>
-            <td>5</td>
-            <td>5</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_67eef223bccf445494d5d1e2980f92c4.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td></td>
-            <td></td>
-            <td>5</td>
-            <td></td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Thank you for your response, Tony. However, I was expecting more detailed information about the total cost of ownership for your solution. Can you provide further clarification on that?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response was vague and did not provide any numerical or contextual information about the total cost of ownership. It appears as a misunderstanding or an error in communication.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_6eb00dacfbce452695749b54a9473bda.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>The response provided does not contain any relevant information regarding the total cost of ownership for the solution. It seems to be incomplete or not related to the question asked.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_6f0e941e245e4342a7d7742c96056e3e.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, I'm doing well, thank you! I have a question: What's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Thank you for the information, Tony! It helps to have clarity on the cost.</td>
-            <td></td>
-            <td>5</td>
-            <td>The answer was straightforward and provided the requested information clearly. However, it lacks additional context or breakdown regarding what the cost includes, which could have been beneficial.</td>
-            <td>5</td>
-            <td>5</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_8be5d87c4f254223886d6fc6fd20eb35.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td></td>
-            <td></td>
-            <td>5</td>
-            <td></td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Thank you for your response, Tony. However, I was expecting more detailed information about the total cost of ownership for your solution. Can you provide further clarification on that?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response was vague and did not provide any numerical or contextual information about the total cost of ownership. It appears as a misunderstanding or an error in communication.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_948bfae4ec42480fad75b2e0cdd3591d.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>0</td>
-            <td></td>
-            <td></td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>dsf</td>
-            <td>Our customers have to consider annual running costs (consumption), one time implementation fee, training costs, and support costs as they come on to our platform. Customers have cited productivity gains of 25% and analytics costs going down by 30% within 18 months post implementation.  </td>
-            <td>1</td>
-            <td>The response did not provide any relevant information regarding the total cost of ownership for the solution. Therefore, I cannot consider this a satisfactory answer.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_acd7a6027b0d4e2397fb99ce5fa0861c.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, thanks for reaching out! I'm interested to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, I appreciate your response, but it seems I didn't get the specific details I was looking for regarding the total cost of ownership for your solution. Could you provide me with a breakdown of that information?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response did not provide any useful information about the total cost of ownership. It was vague and unhelpful.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_bfc79abe9a474092a98769c9ba96d71d.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hello Tony! I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>The response provided does not contain any relevant information regarding the total cost of ownership for the solution. It seems to be incomplete or not related to the question asked.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

dashboards/dashboard_f00a8bbcff044f708de1daf93a73d74f.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>0</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td></td>
-            <td></td>
-            <td>Our customers have to consider annual running costs (consumption), one time implementation fee, training costs, and support costs as they come on to our platform. Customers have cited productivity gains of 25% and analytics costs going down by 30% within 18 months post implementation.  </td>
-            <td>1</td>
-            <td>The response was not helpful or informative. It provided no relevant information regarding the total cost of ownership, which is crucial for my decision-making process.</td>
-            <td>5</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

data/zzzSalesBuddy_Opportunities.csv

@@ -1,7 +0,0 @@
-Opportunity ID,Customer Name,Opportunity Name,Opportunity Stage,Opportunity Description,Opportunity Value,Close Date,Customer Contact,Customer Contact Role,Activity,Next Steps
-XFR0001,HSBC,Lending - Loan Origination System,Proposal,Developing analytic capabilities for the loan origination system,"$250,000",11/30/2024,John Smith,"VP, Information Technology",Met John on 9/16. Product demo went well and pricing was submitted. We have discussed the value proposition of our solution and John's team agrees that our solution's real time analytic capabilities will provide them the ability to improve their credit decisioning process,Meet with John on 10/12 to discuss pricing
-XFR0002,Capital One,,,,,,,,,
-XFR0003,HSBC,,,,,,,,,
-XFR0004,PNC Bank,,,,,,,,,
-XFR0005,,,,,,,,,,
-XFR0006,,,,,,,,,,

requirements.txt

@@ -9,4 +9,9 @@ comtypes==1.2.0
 pandas==2.1.4
 tabulate==0.9.0
 gdown==4.7.1
-Jinja2==3.1.2
+Jinja2==3.1.2
+datasets==2.14.5
+ragas==0.1.0
+nltk==3.9.1
+rouge-score==0.1.2
+sentence-transformers==2.3.0

static/dashboards/dashboard_0ec0b22e698c4db19e140257d9fa0f56.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, thanks for reaching out! Could you tell me what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, thanks for your response! I noticed that you didn't provide any specific details about the total cost of ownership. Can you clarify that for me?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response did not provide any useful information regarding the total cost of ownership, which is crucial for my decision-making process. The lack of clarity is disappointing.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

static/dashboards/dashboard_17b1f47a0e424c8c98551303bc9f44c5.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hello Tony, I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hello Tony, I hope you're doing well. I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response was unhelpful and did not address my question about the total cost of ownership. It seems that there was a misunderstanding or an error in communication.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

static/dashboards/dashboard_2544c3a4c60249f4a231eac133e8e9e8.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, nice to connect! I wanted to ask, what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td></td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, nice to connect! I wanted to ask, what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response was not satisfactory as it did not provide any information regarding the total cost of ownership. It simply stated '434', which is unclear and unhelpful. This leaves me without the necessary information to evaluate the solution's affordability and overall value.</td>
-            <td>4</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

static/dashboards/dashboard_ca3b8724dfce419da3254aadacbc969d.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, thanks for reaching out. I'm interested in understanding the total cost of ownership for your solution.</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, thanks for your response. However, I believe there may have been a misunderstanding regarding the total cost of ownership for your solution. It typically includes not just the initial purchase price but also maintenance, support, and other associated costs over time. Could you clarify that for me?</td>
-            <td></td>
-            <td>1</td>
-            <td>The response provided by the rep is insufficient and overly simplistic. A total cost of ownership should encompass multiple factors, not just a single dollar amount, which indicates a lack of thoroughness and understanding of customer needs.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

static/dashboards/dashboard_f2a671e972a747cdb996dc84a8de9f5a.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        
-        <tr>
-            <td>1</td>
-            <td>What's the total cost of ownership for your solution?</td>
-            <td>Hi Tony, thanks for reaching out! I would like to know what's the total cost of ownership for your solution?</td>
-            <td></td>
-            <td>None</td>
-            <td>None</td>
-            <td>5</td>
-            <td>None</td>
-        </tr>
-        
-        <tr>
-            <td>2</td>
-            <td></td>
-            <td>Hi Tony, thanks for your response. Can you clarify what the total cost of ownership for your solution includes?</td>
-            <td></td>
-            <td>1</td>
-            <td>The answer provided is unclear and does not address my question about the total cost of ownership. It lacks detail and context, making it difficult to evaluate the solution's financial implications.</td>
-            <td>3</td>
-            <td>1</td>
-        </tr>
-        
-    </table>
-</body>
-</html>

templates/dashboard_template.html

@@ -1,52 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Response Dashboard</title>
-    <style>
-        table {
-            border-collapse: collapse;
-            width: 100%;
-        }
-        th, td {
-            border: 1px solid #ddd;
-            padding: 8px;
-            text-align: left;
-        }
-        th {
-            background-color: #f2f2f2;
-        }
-        tr:nth-child(even) {
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Response Dashboard</h1>
-    <table>
-        <tr>
-            <th>#</th>
-            <th>Question</th>
-            <th>Response</th>
-            <th>Ground Truth</th>
-            <th>Score</th>
-            <th>Evaluation</th>
-            <th>Mood Score</th>
-            <th>Overall Evaluation</th>
-        </tr>
-        {% for response in responses %}
-        <tr>
-            <td>{{ response.question_number }}</td>
-            <td>{{ response.question }}</td>
-            <td>{{ response.response }}</td>
-            <td>{{ response.ground_truth }}</td>
-            <td>{{ response.response_score }}</td>
-            <td>{{ response.response_evaluation }}</td>
-            <td>{{ response.mood_score }}</td>
-            <td>{{ response.overall_evaluation }}</td>
-        </tr>
-        {% endfor %}
-    </table>
-</body>
-</html>

utils_data.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import os
 
-
 def get_questions(stage=None, num_questions=0):
     q_and_a_filename = "AIE4_SalesBuddy_QandA.csv" 
     data = read_csv_from_data_folder(q_and_a_filename)
@@ -15,23 +14,13 @@ def get_questions(stage=None, num_questions=0):
         question = {
             'stage': row['Stage'],
             'question': row['Customer Question (Provided by Claude)'],
-            # 'salesperson_response': row['Salesperson Response (Provided by Claude)'],
             'ground_truth': row['Ground Truth (Provided by Nitin). Marked as Salesperson 2 by Claude'],
-            # 'salesperson_rating': row['Rating of Salesperson response (per Claude)'],
-            # 'expert_rating': row['Rating of Groundtruth (per Claude)'],
-            # 'comments': row['Comments (per Claude)']
         }
         if num_questions == 0 or num_questions > len(questions):
             print(f"Adding question: {question['question']}")
             questions.append(question)
     print(f"Returned {len(questions)} questions for stage {stage}")
     return questions
-    # co = [
-    #     {"QID": 1, 'Question': 'What is your product?', 'Ground Truth': 'Our product is a CRM software.'},
-    #     {"QID": 2, 'Question': 'How much does it cost?', 'Ground Truth': 'It costs $99 per month.'},
-    #     {"QID": 3, 'Question': 'What are the key features?', 'Ground Truth': 'Key features include contact management, sales tracking, and reporting.'}
-    # ]
-    # return co
 
 def get_customer_background(scenario, customer_name="HSBC" ):
     background_filename = f"{customer_name}_background.txt"
@@ -40,7 +29,6 @@ def get_customer_background(scenario, customer_name="HSBC" ):
     print(f"Read background for {customer_name} - {background_filename}")
 
 def get_company_data(scenario):
-
     company = {
         "name": "BetterTech",
         "description": """
@@ -140,20 +128,7 @@ def get_opportunities():
     data = read_csv_from_data_folder(opportunities_filename)
     return data
 
-
-
 def read_csv_from_data_folder(filename, handle_nan='drop'):
-    """
-    Read a CSV file from the 'data' folder or Google Drive, and handle NaN values.
-    
-    Args:
-    filename (str): The name of the CSV file to read.
-    google_drive_id (str): The Google Drive file ID. If provided, it will be used if local file is not found.
-    handle_nan (str): Method to handle NaN values. Options: 'drop', 'fill_na', 'fill_empty'. Default is 'drop'.
-    
-    Returns:
-    pandas.DataFrame: The contents of the CSV file as a DataFrame with NaN values handled.
-    """
     data_folder = "./data/"
     file_path = os.path.join(data_folder, filename)
 
@@ -184,15 +159,6 @@ def read_csv_from_data_folder(filename, handle_nan='drop'):
         return None
     
 def read_txt_from_data_folder(filename):
-    """
-    Read a text file from the 'data' folder.
-    
-    Args:
-    filename (str): The name of the text file to read.
-    
-    Returns:
-    str: The contents of the text file as a string.
-    """
     data_folder = "./data/"
     file_path = os.path.join(data_folder, filename)
 
@@ -208,10 +174,4 @@ def read_txt_from_data_folder(filename):
         print(f"An error occurred while reading '{filename}': {str(e)}")
         return None
 
-    
-
-
-if __name__ == "__main__":
-    q_and_a_data = get_q_and_a()
-    if q_and_a_data is not None:
-        print(q_and_a_data.head())
+    

utils_evaluate.py

@@ -1,47 +1,80 @@
-def get_evaluation():
-    out_text = "Evaluation for: Customer/Deal/Scenario"
-    # df = create_sample_data()
-    
-    # # Create a temporary HTML file
-    # with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.html') as f:
-    #     html_content = f"""
-    #     <html>
-    #     <head>
-    #         <style>
-    #             table {{
-    #                 border-collapse: collapse;
-    #                 width: 100%;
-    #                 max-width: 800px;
-    #                 margin: 20px auto;
-    #             }}
-    #             th, td {{
-    #                 border: 1px solid #ddd;
-    #                 padding: 8px;
-    #                 text-align: left;
-    #             }}
-    #             th {{
-    #                 background-color: #f2f2f2;
-    #             }}
-    #             tr:nth-child(even) {{
-    #                 background-color: #f9f9f9;
-    #             }}
-    #         </style>
-    #     </head>
-    #     <body>
-    #         <h2>Evaluation Results</h2>
-    #         {df.to_html(index=False)}
-    #     </body>
-    #     </html>
-    #     """
-    #     f.write(html_content)
-    #     temp_file_path = f.name
-
-    # # Get the file URL
-    # file_url = f"file://{os.path.abspath(temp_file_path)}"
-    # print(file_url)
-    # # Send a message with the link
-    # await cl.Message(content=f"Evaluation Results: [View Table]({file_url})").send()
-    
-    # # Optionally, open the file in the default web browser
-    # webbrowser.open(file_url)
-    return out_text
+import pandas as pd
+from datasets import Dataset
+from nltk.translate.bleu_score import sentence_bleu
+
+from ragas import evaluate
+from ragas.metrics import (
+    answer_relevancy,
+    answer_correctness,
+)
+from rouge_score import rouge_scorer
+from sentence_transformers import SentenceTransformer, util
+
+def evaluate_answers(session):
+    ragas_results = evaluate_with_ragas(session)
+    session.ragas_results = ragas_results
+    scores = []
+    for response in session.responses:
+        bleu_score = calculate_bleu_score(response.get("response", ""), response.get("ground_truth", ""))
+        rouge_score = calculate_rouge_score(response.get("response", ""), response.get("ground_truth", ""))
+        semantic_similarity_score = calculate_semantic_similarity(response.get("response", ""), response.get("ground_truth", ""))
+        all_scores = {
+            "bleu_score": bleu_score,
+            "rouge_score": rouge_score,
+            "semantic_similarity_score": semantic_similarity_score
+        }
+        scores.append(all_scores)
+    session.scores = scores
+    return scores
+
+
+
+def evaluate_with_ragas(session):
+    questions = [] 
+    answers = []
+    ground_truths = []
+    contexts = []
+    for i, response in enumerate(session.responses, 1):
+        questions.append(response.get("question", ""))
+        answers.append(response.get("response", ""))
+        ground_truths.append(response.get("ground_truth", ""))
+        contexts.append([session.company.product_description])
+
+    evaluation_dataset = Dataset.from_dict({
+        "question" : questions,
+        "answer" : answers,
+        "contexts" : contexts,
+        "ground_truth" : ground_truths
+    })
+
+    print(evaluation_dataset)
+
+    metrics = [
+        # faithfulness,
+        answer_relevancy,
+        # context_recall,
+        # context_precision,
+        answer_correctness,
+    ]
+    results = evaluate(evaluation_dataset, metrics)
+    print(results)
+    return results
+
+def calculate_bleu_score(answer, ground_truth):
+    bleu_score = sentence_bleu([ground_truth.split()], answer.split())
+    print(f"BLEU score: {bleu_score}")
+    return bleu_score
+
+def calculate_rouge_score(answer, ground_truth):
+    scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
+    rouge_scores = scorer.score(ground_truth, answer)
+    print(f"ROUGE score: {rouge_scores}")
+    return rouge_scores
+
+def calculate_semantic_similarity(answer, ground_truth):
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    answer_embedding = model.encode(answer)
+    ground_truth_embedding = model.encode(ground_truth)
+    similarity_score = util.cos_sim(answer_embedding, ground_truth_embedding)
+    print(f"Semantic Similarity: {similarity_score.item()}")
+    return similarity_score.item()

utils_output.py

@@ -1,20 +1,114 @@
-import os
-import uuid
-from jinja2 import Environment, FileSystemLoader
-
-def create_html_dashboard(responses, output_dir="dashboards"):
-    unique_filename = f"dashboard_{uuid.uuid4().hex}.html"
-    
-    if not os.path.exists(output_dir):
-        os.makedirs(output_dir)
-    
-    env = Environment(loader=FileSystemLoader('templates'))
-    template = env.get_template('dashboard_template.html')
-
-    html_content = template.render(responses=responses)
-
-    full_path = os.path.join(output_dir, unique_filename)
-    with open(full_path, "w") as f:
-        f.write(html_content)
-
-    return unique_filename  # Return just the filename
+import asyncio
+import json
+import re
+from datetime import datetime
+
+from utils_evaluate import evaluate_answers
+
+async def display_llm_responses(cl, this_session):
+    output = f"**Responses**"
+    await cl.Message(content=output).send()
+    for query, response in zip(this_session.queries, this_session.llm_responses):
+        query_display = {
+            "command": query["command"],
+            "message": query["message"],
+            "mood_score": query["mood_score"],
+            "previous_question": query["previous_question"],
+            "rep_answer": query["rep_answer"],
+            "next_question": query["next_question"],
+        }
+        query_json = json.dumps(query_display, indent=2)
+        await cl.Message(content="Query:").send()
+        await cl.Message(content=query_json).send()
+        await cl.Message(content="Response:").send()
+        await cl.Message(content=response).send()
+
+    remaining_queries = this_session.queries[len(this_session.llm_responses):]
+    remaining_responses = this_session.llm_responses[len(this_session.queries):]
+
+    for query in remaining_queries:
+        await cl.Message(content=f"**Query:** {query}").send()
+
+    for response in remaining_responses:
+        await cl.Message(content=f"**Response:** {response}").send()
+
+def format_score(score):
+    if isinstance(score, (int, float)):
+        return f"{score*100:.1f}%"
+    return score 
+
+def format_rogue_score(score):
+    if isinstance(score, str):
+        match = re.search(r'precision=([\d.]+), recall=([\d.]+), fmeasure=([\d.]+)', score)
+        if match:
+            precision = float(match.group(1))
+            recall = float(match.group(2))
+            fmeasure = float(match.group(3))
+            return f"Precision: {precision*100:.1f}%, Recall: {recall*100:.1f}%, FMeasure: {fmeasure*100:.1f}%"
+    else:
+        precision = score.precision
+        recall = score.recall
+        fmeasure = score.fmeasure
+        return f"Precision: {precision*100:.1f}%, Recall: {recall*100:.1f}%, FMeasure: {fmeasure*100:.1f}%"     
+    return score  #
+
+def format_datetime(dt):
+    if isinstance(dt, datetime):
+        return dt.strftime("%Y-%m-%d %H:%M") 
+    return str(dt)  #
+
+async def display_evaluation_results(cl, this_session):
+    out_text = "All questions answered. Preparing evaluation results..."
+    await cl.Message(content=out_text).send()
+    evaluate_answers(this_session) 
+    await asyncio.sleep(1)
+
+    output = f"**Session Summary**"
+    await cl.Message(content=output).send()
+    output = f"**Start Time:** {format_datetime(this_session.start_time)} \n"
+    output = output + f"**End Time:** {format_datetime(this_session.end_time)} \n"
+    output = output + f"**Duration:** {this_session.duration_minutes} minutes \n"
+    output = output + f"**Total Number of Questions:** {len(this_session.questions)} \n"
+    output = output + f"**Total Questions Answered:** {len(this_session.responses)} \n"
+    await cl.Message(content=output).send()
+
+    results_df = this_session.ragas_results.to_pandas()
+    columns_to_average = ['answer_relevancy', 'answer_correctness']
+    averages = results_df[columns_to_average].mean()
+
+    await cl.Message(content="**Overall Summary (By SalesBuddy)**").send()
+    output = f"**Overall Score:** {this_session.responses[-1]['overall_score']} \n"
+    output = output + f"**Overall Evaluation:** {this_session.responses[-1]['overall_evaluation']} \n"
+    output = output + f"**Final Mood Score:** {this_session.responses[-1]['mood_score']} \n"
+    await cl.Message(content=output).send()
+
+    await cl.Message(content="**Average Scores - Based on RAGAS**").send()
+
+    output = "Answer Relevancy: " + str(format_score(averages['answer_relevancy'])) + "\n"
+    output = output + "Answer Correctness: " + str(format_score(averages['answer_correctness'])) + "\n"
+    await cl.Message(content=output).send()
+
+    await cl.Message(content="**Individual Question Scores**").send()
+
+    for index, resp in enumerate(this_session.responses):
+        scores = this_session.scores[index]
+        relevancy = results_df.iloc[index].get('answer_relevancy', 'N/A')
+        correctness = results_df.iloc[index].get('answer_correctness', 'N/A')
+        bleu_score = scores.get('bleu_score', 'N/A')
+        rouge1_score = scores.get('rouge_score', {}).get('rouge1', 'N/A')
+        rouge1_output = format_rogue_score(rouge1_score)
+        rougeL_score = scores.get('rouge_score', {}).get('rougeL', 'N/A')
+        rougeL_output = format_rogue_score(rougeL_score)
+        semantic_similarity_score = scores.get('semantic_similarity_score', 'N/A')
+        output = f"""
+            **Question:** {resp.get('question', 'N/A')}
+            **Answer:** {resp.get('response', 'N/A')}
+            **Ground Truth:** {resp.get('ground_truth', 'N/A')}     
+            **Answer Relevancy:** {format_score(relevancy)}
+            **Answer Correctness:** {format_score(correctness)}
+            **BLEU Score:** {format_score(bleu_score)}
+            **ROUGE 1 Score:** {rouge1_output}
+            **ROUGE L Score:** {rougeL_output}
+            **Semantic Similarity Score:** {format_score(semantic_similarity_score)}
+        """ 
+        await cl.Message(content=output).send()

utils_prompt.py

@@ -1,6 +1,40 @@
 from langchain_core.prompts import ChatPromptTemplate
 
 def get_user_template():
+    # user_template = get_user_template_openai_long()
+    user_template = get_user_template_openai_short()
+    return user_template
+
+def get_user_template_openai_short():
+    user_template = """
+        Conversation Mode:
+        {conversation_mode}
+
+        Name:
+        {name}  
+
+        Sales Rep:
+        {sales_rep}  
+
+        Command:
+        {command}
+
+        Next Question:
+        {next_question}
+
+        Previous Question:
+        {previous_question} 
+
+        Message:
+        {message}
+
+        Rep Answer:
+        {rep_answer}
+        
+    """
+    return user_template
+
+def get_user_template_openai_long():
     user_template = """
         Conversation Mode:
         {conversation_mode}
@@ -32,6 +66,9 @@ def get_user_template():
         Stage:
         {stage}     
 
+        Command:
+        {command}
+
         Next Question:
         {next_question}
 
@@ -46,16 +83,49 @@ def get_user_template():
         
         Conversation History:
         {conversation_history}
-
-
-
     """
     return user_template
 
 
+def get_system_template():
+    # system_template = get_system_template_openai_long()
+    system_template = get_system_template_openai_short()
+    return system_template
 
+def get_system_template_openai_short(): 
+    system_template = """
+        You are playing a role in a conversation with a sales representative.
+        Your name is in the 'Name:' section.
+        If the sales rep gets your name wrong, you can correct them, politely
+        They can use your first name, full name or address you with a title and last name.
+        Your name does not need to match exactly what they say.
+        Your compnay information is in the 'Company:' section.
+        The sales rep's details is in the 'Sales rep:' section.
+        You do not need to use the sales rep's name in your response except in the greeting.
+        The sales rep's company information is in the 'Rep company:' section.
+        You are to have a conversation with the sales rep and evaluate their responses.
+        The previous question you asked is in the 'Previous question:' section.
+        The rep's answer to the previous question is in the 'Rep answer:' section.
+        You are given a command in the 'Command:' section.
+        You can make conversation but you must follow the command.
+        If a previous question and answer are provided, you must evaluate the rep's answer.
+        You will perform evaluation based on how well and thoroughly the rep answered the previous question.
+        You will ALWAYS provide your response in valid JSON format
+        Remember all string values must be enclosed in double quotes.
+        You will include with the following fields in JSON format:
+        - Continue: Yes or No depending on if you want to continue the conversation based on the reps answer to your question.
+        - Ask Follow Up: Yes or No depending on if you want to ask a follow up question.
+        - Response: Your response to the message but do not include a question in the response.
+        - Question: The next question to ask.
+        - Score: A score from 1 to 10 based on how well the rep answered your previous question.
+        - Evaluation: A evaluation of the rep based on their answer to your previous question.
+        - Mood Score: A score from 1 to 10 based on how you feel the conversation is going.
+        - Overall Score: A score from 1 to 10 the rep based on all of their answers to your questions.
+        - Overall Evaluation: A text evaluation of the rep based on all of their answers to your questions.
+    """
+    return system_template
 
-def get_system_template():
+def get_system_template_openai_long():  
     system_template = """
         You playing a role in a conversation with a sales representative.
         You are a customer of the sales rep's company.
@@ -125,9 +195,8 @@ def get_system_template():
         Stop asking follow up questions when the number of follow up questions reaches 5 or the rep says they can't answer the question.
         If the rep says they can't answer the question, set the score to a 1.   
         If they have satisfied your questions thank them.
-        
-
-        
+        REMEMBER NEVER ASK FOLLOW UP QUESTIONS IF THE CONVERSATION MODE IS SINGLE.
+        The Command: section will tell you if you can ask an original question or a follow up question or end the session.       
     """ 
     return system_template