notebooks/rag_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import minsearch\n",
    "from groq import Groq\n",
    "from mistralai import Mistral\n",
    "from dotenv import load_dotenv\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Environment Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Access the API keys\n",
    "mistral_api_key = os.getenv(\"MISTRAL_API_KEY\")\n",
    "groq_api_key = os.getenv(\"GROQ_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Groq(api_key=groq_api_key)\n",
    "client_mistral=Mistral(api_key=mistral_api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"../dataset/data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question_ID</th>\n",
       "      <th>Questions</th>\n",
       "      <th>Answers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1590140</td>\n",
       "      <td>What does it mean to have a mental illness?</td>\n",
       "      <td>Mental illnesses are health conditions that di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2110618</td>\n",
       "      <td>Who does mental illness affect?</td>\n",
       "      <td>It is estimated that mental illness affects 1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6361820</td>\n",
       "      <td>What causes mental illness?</td>\n",
       "      <td>It is estimated that mental illness affects 1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9434130</td>\n",
       "      <td>What are some of the warning signs of mental i...</td>\n",
       "      <td>Symptoms of mental health disorders vary depen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7657263</td>\n",
       "      <td>Can people with mental illness recover?</td>\n",
       "      <td>When healing from mental illness, early identi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1619387</td>\n",
       "      <td>What should I do if I know someone who appears...</td>\n",
       "      <td>Although this website cannot substitute for pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1030153</td>\n",
       "      <td>How can I find a mental health professional fo...</td>\n",
       "      <td>Feeling comfortable with the professional you ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8022026</td>\n",
       "      <td>What treatment options are available?</td>\n",
       "      <td>Just as there are different types of medicatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1155199</td>\n",
       "      <td>If I become involved in treatment, what do I n...</td>\n",
       "      <td>Since beginning treatment is a big step for in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7760466</td>\n",
       "      <td>What is the difference between mental health p...</td>\n",
       "      <td>There are many types of mental health professi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Question_ID                                          Questions  \\\n",
       "0      1590140        What does it mean to have a mental illness?   \n",
       "1      2110618                    Who does mental illness affect?   \n",
       "2      6361820                        What causes mental illness?   \n",
       "3      9434130  What are some of the warning signs of mental i...   \n",
       "4      7657263            Can people with mental illness recover?   \n",
       "5      1619387  What should I do if I know someone who appears...   \n",
       "6      1030153  How can I find a mental health professional fo...   \n",
       "7      8022026              What treatment options are available?   \n",
       "8      1155199  If I become involved in treatment, what do I n...   \n",
       "9      7760466  What is the difference between mental health p...   \n",
       "\n",
       "                                             Answers  \n",
       "0  Mental illnesses are health conditions that di...  \n",
       "1  It is estimated that mental illness affects 1 ...  \n",
       "2  It is estimated that mental illness affects 1 ...  \n",
       "3  Symptoms of mental health disorders vary depen...  \n",
       "4  When healing from mental illness, early identi...  \n",
       "5  Although this website cannot substitute for pr...  \n",
       "6  Feeling comfortable with the professional you ...  \n",
       "7  Just as there are different types of medicatio...  \n",
       "8  Since beginning treatment is a big step for in...  \n",
       "9  There are many types of mental health professi...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 98 entries, 0 to 97\n",
      "Data columns (total 3 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   Question_ID  98 non-null     int64 \n",
      " 1   Questions    98 non-null     object\n",
      " 2   Answers      98 non-null     object\n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 2.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents=df.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Question_ID': 1590140,\n",
       " 'Questions': 'What does it mean to have a mental illness?',\n",
       " 'Answers': 'Mental illnesses are health conditions that disrupt a person’s thoughts, emotions, relationships, and daily functioning. They are associated with distress and diminished capacity to engage in the ordinary activities of daily life.\\nMental illnesses fall along a continuum of severity: some are fairly mild and only interfere with some aspects of life, such as certain phobias. On the other end of the spectrum lie serious mental illnesses, which result in major functional impairment and interference with daily life. These include such disorders as major depression, schizophrenia, and bipolar disorder, and may require that the person receives care in a hospital.\\nIt is important to know that mental illnesses are medical conditions that have nothing to do with a person’s character, intelligence, or willpower. Just as diabetes is a disorder of the pancreas, mental illness is a medical condition due to the brain’s biology.\\nSimilarly to how one would treat diabetes with medication and insulin, mental illness is treatable with a combination of medication and social support. These treatments are highly effective, with 70-90 percent of individuals receiving treatment experiencing a reduction in symptoms and an improved quality of life. With the proper treatment, it is very possible for a person with mental illness to be independent and successful.'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indexing the data Using Minsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
      "100  3832  100  3832    0     0   4800      0 --:--:-- --:--:-- --:--:--  4838\n",
      "100  3832  100  3832    0     0   4785      0 --:--:-- --:--:-- --:--:--  4820\n"
     ]
    }
   ],
   "source": [
    "!curl -O https://raw.githubusercontent.com/DataTalksClub/llm-zoomcamp/main/01-intro/minsearch.py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question_ID</th>\n",
       "      <th>Questions</th>\n",
       "      <th>Answers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1590140</td>\n",
       "      <td>What does it mean to have a mental illness?</td>\n",
       "      <td>Mental illnesses are health conditions that di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2110618</td>\n",
       "      <td>Who does mental illness affect?</td>\n",
       "      <td>It is estimated that mental illness affects 1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6361820</td>\n",
       "      <td>What causes mental illness?</td>\n",
       "      <td>It is estimated that mental illness affects 1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9434130</td>\n",
       "      <td>What are some of the warning signs of mental i...</td>\n",
       "      <td>Symptoms of mental health disorders vary depen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7657263</td>\n",
       "      <td>Can people with mental illness recover?</td>\n",
       "      <td>When healing from mental illness, early identi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>4373204</td>\n",
       "      <td>How do I know if I'm drinking too much?</td>\n",
       "      <td>Sorting out if you are drinking too much can b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>7807643</td>\n",
       "      <td>If cannabis is dangerous, why are we legalizin...</td>\n",
       "      <td>Cannabis smoke, for example, contains cancer-c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>4352464</td>\n",
       "      <td>How can I convince my kids not to use drugs?</td>\n",
       "      <td>You can't. But you can influence their capacit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>6521784</td>\n",
       "      <td>What is the legal status (and evidence) of CBD...</td>\n",
       "      <td>Cannabidiol or CBD is a naturally occurring co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>3221856</td>\n",
       "      <td>What is the evidence on vaping?</td>\n",
       "      <td>\"Vaping\" is the term for using a device where ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Question_ID                                          Questions  \\\n",
       "0       1590140        What does it mean to have a mental illness?   \n",
       "1       2110618                    Who does mental illness affect?   \n",
       "2       6361820                        What causes mental illness?   \n",
       "3       9434130  What are some of the warning signs of mental i...   \n",
       "4       7657263            Can people with mental illness recover?   \n",
       "..          ...                                                ...   \n",
       "93      4373204            How do I know if I'm drinking too much?   \n",
       "94      7807643  If cannabis is dangerous, why are we legalizin...   \n",
       "95      4352464       How can I convince my kids not to use drugs?   \n",
       "96      6521784  What is the legal status (and evidence) of CBD...   \n",
       "97      3221856                    What is the evidence on vaping?   \n",
       "\n",
       "                                              Answers  \n",
       "0   Mental illnesses are health conditions that di...  \n",
       "1   It is estimated that mental illness affects 1 ...  \n",
       "2   It is estimated that mental illness affects 1 ...  \n",
       "3   Symptoms of mental health disorders vary depen...  \n",
       "4   When healing from mental illness, early identi...  \n",
       "..                                                ...  \n",
       "93  Sorting out if you are drinking too much can b...  \n",
       "94  Cannabis smoke, for example, contains cancer-c...  \n",
       "95  You can't. But you can influence their capacit...  \n",
       "96  Cannabidiol or CBD is a naturally occurring co...  \n",
       "97  \"Vaping\" is the term for using a device where ...  \n",
       "\n",
       "[98 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Question_ID': 'question_id', 'Questions': 'questions','Answers':'answers'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['question_id', 'questions', 'answers'], dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents=df.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question_id': 1590140,\n",
       " 'questions': 'What does it mean to have a mental illness?',\n",
       " 'answers': 'Mental illnesses are health conditions that disrupt a person’s thoughts, emotions, relationships, and daily functioning. They are associated with distress and diminished capacity to engage in the ordinary activities of daily life.\\nMental illnesses fall along a continuum of severity: some are fairly mild and only interfere with some aspects of life, such as certain phobias. On the other end of the spectrum lie serious mental illnesses, which result in major functional impairment and interference with daily life. These include such disorders as major depression, schizophrenia, and bipolar disorder, and may require that the person receives care in a hospital.\\nIt is important to know that mental illnesses are medical conditions that have nothing to do with a person’s character, intelligence, or willpower. Just as diabetes is a disorder of the pancreas, mental illness is a medical condition due to the brain’s biology.\\nSimilarly to how one would treat diabetes with medication and insulin, mental illness is treatable with a combination of medication and social support. These treatments are highly effective, with 70-90 percent of individuals receiving treatment experiencing a reduction in symptoms and an improved quality of life. With the proper treatment, it is very possible for a person with mental illness to be independent and successful.'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "index=minsearch.Index(\n",
    "    text_fields=['questions', 'answers'],\n",
    "    keyword_fields=[]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<minsearch.Index at 0x1dd02704e80>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.fit(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "query=\"What should I eat if I lost a friend\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'question_id': 4759773,\n",
       "  'questions': 'What should I do if I’m worried about a friend or relative?',\n",
       "  'answers': 'This may depend on your relationship with them. Gently encouraging someone to seek appropriate support would be helpful to start with.'},\n",
       " {'question_id': 3388962,\n",
       "  'questions': 'What should I know before starting a new medication?',\n",
       "  'answers': 'The best source of information regarding medications is the physician prescribing them. He or she should be able to answer questions such as:    1. What is the medication supposed to do? 2. When should it begin to take effect, and how will I know when it is effective? 3. How is the medication taken and for how long? What food, drinks, other medicines, and activities should be avoided while taking this medication? 4. What are the side effects and what should be done if they occur? 5. What do I do if a dose is missed? 6. Is there any written information available about this medication? 7. Are there other medications that might be appropriate? 8. If so, why do you prefer the one you have chosen? 9. How do you monitor medications and what symptoms indicate that they should be raised, lowered, or changed? 10. All medications should be taken as directed. Most medications for mental illnesses do not work when taken irregularly, and extra doses can cause severe, sometimes dangerous side effects. Many psychiatric medications begin to have a beneficial effect only after they have been taken for several weeks.'},\n",
       " {'question_id': 1619387,\n",
       "  'questions': 'What should I do if I know someone who appears to have the symptoms of a mental disorder?',\n",
       "  'answers': \"Although this website cannot substitute for professional advice, we encourage those with symptoms to talk to their friends and family members and seek the counsel of a mental health professional. The sooner the mental health condition is identified and treated, the sooner they can get on the path to recovery.\\nIf you know someone who is having problems, don't assume that the issue will resolve itself. Let them know that you care about them, and that there are treatment options available that will help them heal. Speak with a mental health professional or counselor if you think your friend or family member is experiencing the symptoms of a mental health condition. If the affected loved one knows that you support them, they will be more likely to seek out help.\"},\n",
       " {'question_id': 8690253,\n",
       "  'questions': 'What do I do if I’m worried about my mental health?',\n",
       "  'answers': 'The most important thing is to talk to someone you trust. This might be a friend, colleague, family member, or GP. In addition to talking to someone, it may be useful to find out more information about what you are experiencing. These things may help to get some perspective on what you are experiencing, and be the start of getting help.'},\n",
       " {'question_id': 1259439,\n",
       "  'questions': 'If I become involved in treatment what do I need to know?',\n",
       "  'answers': 'Beginning treatment is a big step for individuals and families and can be very overwhelming. It is important to continue involvement in the treatment process as much as possible. Some questions you will need to have answered include:\\nWhat is known about the cause of this particular illness?\\nAre there other diagnoses where these symptoms are common?\\nDo you normally include a physical or neurological examination?\\nAre there any additional tests or exams that you would recommend at this point?\\nWould you advise an independent opinion from another psychiatrist at this point?\\nWhat program of treatment is the most helpful with this diagnosis?\\nWill this program involve services by other specialists? If so, who will be responsible for coordinating these services?\\nWhat do you see as the family’s role in this program of treatment?\\nHow much access will the family have to the individuals who are providing the treatment?\\nWhat medications are generally used with this diagnosis? What is the biological effect of this medication, and what do you expect it to accomplish? What are the risks associated with the medication? How soon will we be able to tell if the medication is effective, and how will we know?\\nHow much experience do you have in treating individuals with this illness?\\nWhat can I do to help you in the treatment?'},\n",
       " {'question_id': 2973656,\n",
       "  'questions': 'How do I know if I’m unwell?',\n",
       "  'answers': 'If your beliefs , thoughts , feelings or behaviours have a significant impact on your ability to function in what might be considered a normal or ordinary way, it would be important to seek help.'},\n",
       " {'question_id': 9100298,\n",
       "  'questions': 'How can I maintain social connections? What if I feel lonely?',\n",
       "  'answers': \"A lot of people are alone right now, but we don't have to be lonely. We're all in this together. \\n While you may be physically separated from friends, family members, and other loved ones, it has never been more important to maintain those social connections. Social connections are an opportunity to seek and share support, talk through difficult feelings, share a laugh, keep up-to-date with loved ones, and help each other cope. This pandemic is a lot for one person to deal with on their own. While measures like physical distancing and self-isolation are necessary to slow the spread of the virus, the physical separation can amplify a lot of challenging emotions like loneliness and fear. \\n Think about the different ways to connect that are most meaningful for you. For example, you might prefer a video chat over a phone call, or you might prefer to text throughout the day rather than one set time for a video call. Then, work with your social networks to make a plan. You might video chat with your close friends in the evening and phone a family member once a week. \\n Remember to be mindful of people who may not be online. Check in by phone and ask how you can help. \\n The quality of your social connections matter. Mindlessly scrolling through social media and liking a few posts usually doesn't build strong social connections. Make sure you focus on strategies that actually make you feel included and connected. If your current strategies don't help you feel connected, problem-solve to see if you can find a solution. \\n Everyone feels lonely at times—maybe you recently moved to a new city, are changing your circle of friends, lost someone important in your life, or lost your job and also lost important social connections with coworkers. Other people may have physical connections to others but may feel like their emotional or social needs aren't met. Measures like social distancing or self-isolation can make loneliness feel worse no matter why you feel lonely now. \\n Reach out to the connections you do have. Suggest ways to keep in touch and see if you can set a regular time to connect. People may hesitate to reach out for a lot of different reasons, so don't be afraid to be the one who asks. \\n Look for local community support groups and mutual aid groups on social media. This pandemic is bringing everyone together, so look for opportunities to make new connections. These groups are a great way to share your skills and abilities or seek help and support. \\n Look for specialized support groups. Support groups are moving online, and there are a lot of different support lines to call if you need to talk to someone. To find community services in BC, call or text 211 or visit www.bc211.ca. \\n If you need extra support, you can talk with a psychologist or counsellor for free: \\n You can access a free phone call with a Registered Psychologist though the Covid-19 Psychological Support Service from the BC Psychological Association. Visit www.psychologists.bc.ca/covid-19-resources. \\n You can access free, phone-based, short-term support with a counsellor from a new group called the BC COVID-19 Mental Health Network. Email [email protected] to receive an appointment time. \\n For youth people ages 12-24, you can talk with a counsellor for free through Foundry Virtual. Visit foundrybc.ca/get-support/virtual/. \\n Call the BC Mental Health Support Line at 310-6789. It’s available 24/7. \\n Chat online with a Crisis Center volunteer at www.crisiscentrechat.ca (daily between noon and 1:00am) \\n For older adults: Call the Seniors Distress Line at 604-872-123 \\n For youth and young adults: Chat online with a volunteer at www.YouthinBC.com (daily between noon and 1:00am) \\n For children and youth: Call the Kids Help Phone at 1-800-668-6868 or visit kidshelpphone.ca \\n For tips on managing loneliness, check out the following resources: \\n Coping with Loneliness from the Canadian Mental Health Association: cmha.bc.ca/documents/coping-with-loneliness/ \\n Loneliness and Social Connection issue of Visions Journal at www.heretohelp.bc.ca/visions/loneliness-and-social-connection-vol14 \\n Wellness Module 3: Social Support at www.heretohelp.bc.ca/wellness-module/wellness-module-3-social-support\"},\n",
       " {'question_id': 1155199,\n",
       "  'questions': 'If I become involved in treatment, what do I need to know?',\n",
       "  'answers': 'Since beginning treatment is a big step for individuals and families, it can be very overwhelming. It is important to be as involved and engaged in the treatment process as possible. Some questions you will need to have answered include:\\nWhat is known about the cause of this particular illness?\\nAre there other diagnoses where these symptoms are common?\\nDo you normally include a physical or neurological examination?\\nAre there any additional tests or exams that you would recommend at this point?\\nWould you advise an independent opinion from another psychiatrist at this point?\\nWhat program of treatment is the most helpful with this diagnosis?\\nWill this program involve services by other specialists? If so, who will be responsible for coordinating these services?\\nWhat do you see as the family’s role in this program of treatment?\\nHow much access will the family have to the individuals who are providing the treatment?\\nWhat medications are generally used with this diagnosis?\\nHow much experience do you have in treating individuals with this illness?\\nWhat can I do to help you in the treatment?'},\n",
       " {'question_id': 2447683,\n",
       "  'questions': 'Cannabis is legally allowed to 19+ but there are doctor groups saying it’s potentially harmful to age 25. Any use or certain use? What’s myth and what’s fact? If I’m a parent, what should I tell my young adult?',\n",
       "  'answers': 'Using cannabis has the potential for benefits and harms. Young people use cannabis, like other psychoactive drugs, to feel good, to feel better, to do better or to explore. Trying cannabis out of curiosity, as an experiment, or while socializing with friends, is related to moderate use and lower potential for harm. Using cannabis to cope with daily life, deal with unpleasant feelings, or fit in with a social group has higher potential for harm. This is because dealing with these kinds of issues is associated with frequent and heavier use, less thought about potential harms and little consideration of alternatives for coping such as talking with a parent or trusted adult or physical activity with friends. \\n Evidence suggests that the younger a person is when they start using cannabis and the more often they use, the greater the potential for harms. The legal age to use cannabis in BC is 19. However, our brains do not finish developing until about age 25. Delaying cannabis use until early adulthood may reduce potential harmful effects on the brain. \\n Some young people, especially those with many factors predisposing them to serious and persistent mental health issues, should probably not use cannabis. Cannabis has been associated with an increased risk for psychosis and schizophrenia in this small group of people. Some people with serious mental health issues have also reported that using cannabis has helped them cope with their illness by helping them feel less anxious or stressed. As in most situations, balancing potential benefits and harms of using cannabis will be key for young people who have serious mental health concerns. \\n Mixing drugs, such as cannabis and alcohol, can also increase the possibility of experiencing harms. Intoxication may be more intense and long lasting and the young person may not appreciate how impaired they are. We often suggest, “Not too much, not too often, and only in a safe context” as a simple way to gauge your use of any psychoactive substance. \\n As a parent or caring adult, an open respectful relationship with a young person is one of your best resources and ways to prevent harms from substance use. Letting the youth know they can approach you at any time to talk about cannabis, other substances, or anything else of concern to them, says they matter to you and you are ready to listen and engage in dialogue with them. This is a great place to begin addressing anything that might come the young person’s way in life! \\n The Canadian Institute for Substance Use Research, formerly CARBC, is a member of the BC Partners for Mental Health and Addictions Information. The institute is dedicated to the study of substance use in support of community-wide efforts aimed at providing all people with access to healthier lives, whether using substances or not. For more, visit www.cisur.ca.'},\n",
       " {'question_id': 1337085,\n",
       "  'questions': 'I have thoughts of suicide, or someone I care about is talking about suicide. What should I do?',\n",
       "  'answers': 'If you need to talk to someone or you aren’t sure how to help someone you care about, call 1-800-SUICIDE (1-800-784-2433) at any time. Or type your concern using live chat (like texting online) at www.crisiscentrechat.ca between noon and 1am. They can help you, and they can suggest good local resources. If you’re at risk of harm or think someone else is in danger and you need help right now, call 911. \\n It’s scary to have thoughts of suicide or hear that someone you can care about has thoughts of suicide. Thoughts of suicide don’t mean that someone will end their life, but it’s a sign that they need extra help or support. If you have thoughts of suicide, it’s important to talk with your doctor or mental health service provider. If you’re supporting someone else, encourage them to seek help. \\n Coping With Suicidal Thoughts is a good resource to help you understand and manage difficult feelings. \\n Our info sheet on suicide has information on suicide, helping someone else, and finding help. What is Suicide? is a booklet with audio in plain language for lower literacy readers. \\n The Centre for Suicide Prevention has many resource toolkits on suicide for different audiences, including people serving in the military, young people, teens, older adults, Aboriginal community members, and LGBT community members.'}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.search(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_questions = pd.read_csv('../dataset/ground_truth_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1590140</td>\n",
       "      <td>How do mental illnesses affect a person's dail...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1590140</td>\n",
       "      <td>What are some examples of serious mental illne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1590140</td>\n",
       "      <td>Why is it a misconception to associate mental ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1590140</td>\n",
       "      <td>How are mental illnesses treated, and what is ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1590140</td>\n",
       "      <td>Can a person with mental illness become indepe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                           question\n",
       "0  1590140  How do mental illnesses affect a person's dail...\n",
       "1  1590140  What are some examples of serious mental illne...\n",
       "2  1590140  Why is it a misconception to associate mental ...\n",
       "3  1590140  How are mental illnesses treated, and what is ...\n",
       "4  1590140  Can a person with mental illness become indepe..."
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth=df_questions.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 1590140,\n",
       " 'question': \"How do mental illnesses affect a person's daily functioning and relationships?\"}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(query):\n",
    "    boost = {}\n",
    "\n",
    "    results = index.search(\n",
    "        query=query,\n",
    "        filter_dict={},\n",
    "        boost_dict=boost,\n",
    "        num_results=10\n",
    "    )\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hit_rate(relevance_total):\n",
    "    cnt = 0\n",
    "\n",
    "    for line in relevance_total:\n",
    "        if True in line:\n",
    "            cnt = cnt + 1\n",
    "\n",
    "    return cnt / len(relevance_total)\n",
    "\n",
    "def mrr(relevance_total):\n",
    "    total_score = 0.0\n",
    "\n",
    "    for line in relevance_total:\n",
    "        for rank in range(len(line)):\n",
    "            if line[rank] == True:\n",
    "                total_score = total_score + 1 / (rank + 1)\n",
    "\n",
    "    return total_score / len(relevance_total)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision(relevance_list):\n",
    "    \"\"\"\n",
    "    Precision: Proportion of retrieved documents that are relevant.\n",
    "    \"\"\"\n",
    "    relevant_retrieved = sum(relevance_list)  # True indicates relevance\n",
    "    total_retrieved = len(relevance_list)  # All retrieved documents\n",
    "    if total_retrieved == 0:\n",
    "        return 0.0  # Avoid division by zero\n",
    "    return relevant_retrieved / total_retrieved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall(relevance_list, total_relevant):\n",
    "    \"\"\"\n",
    "    Recall: Proportion of relevant documents that are retrieved.\n",
    "    \"\"\"\n",
    "    relevant_retrieved = sum(relevance_list)  # True indicates relevance\n",
    "    if total_relevant == 0:\n",
    "        return 0.0  # Avoid division by zero\n",
    "    return relevant_retrieved / total_relevant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(ground_truth, search_function):\n",
    "    relevance_total = []\n",
    "    precision_scores = []\n",
    "    recall_scores = []\n",
    "\n",
    "    for q in tqdm(ground_truth):\n",
    "        doc_id = q['id']\n",
    "        \n",
    "        # Get search results for this query\n",
    "        results = search_function(q)\n",
    "        \n",
    "        # Check if the correct document (matching question_id) is in the results\n",
    "        relevance = [doc['question_id'] == doc_id for doc in results]\n",
    "        relevance_total.append(relevance)\n",
    "        \n",
    "        # Precision: Fraction of retrieved documents that are relevant\n",
    "        precision_score = precision(relevance)\n",
    "        precision_scores.append(precision_score)\n",
    "        \n",
    "        # Recall: There is only 1 relevant document per query, so recall is either 1 or 0\n",
    "        recall_score = recall(relevance, 1)\n",
    "        recall_scores.append(recall_score)\n",
    "\n",
    "    return {\n",
    "        'hit_rate': hit_rate(relevance_total),\n",
    "        'mrr': mrr(relevance_total),\n",
    "        'precision': sum(precision_scores) / len(precision_scores),  # Average precision\n",
    "        'recall': sum(recall_scores) / len(recall_scores),  # Average recall\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a77c76ffd2434de9ac4b888899b38f21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'hit_rate': 0.9180327868852459,\n",
       " 'mrr': 0.7107739179460493,\n",
       " 'precision': 0.09180327868852516,\n",
       " 'recall': 0.9180327868852459}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(ground_truth, lambda q: search(q['question']))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "High Recall, Low Precision: This suggests that the system is finding most of the relevant documents, but it is also retrieving a lot of irrelevant ones. You might want to focus on improving the precision by better tuning your search algorithm (e.g., improving query relevance, adjusting boosts, or filtering irrelevant results)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking FOr the Best Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.auto import tqdm\n",
    "from typing import Dict, List, Any\n",
    "from itertools import product\n",
    "\n",
    "def calculate_metrics(relevance_total: List[List[bool]]) -> Dict[str, float]:\n",
    "    \"\"\"Calculate all retrieval metrics from relevance lists\"\"\"\n",
    "    \n",
    "    # Calculate hit rate\n",
    "    hit_rate = sum(any(line) for line in relevance_total) / len(relevance_total)\n",
    "    \n",
    "    # Calculate MRR\n",
    "    mrr_score = 0.0\n",
    "    for line in relevance_total:\n",
    "        for rank, relevant in enumerate(line):\n",
    "            if relevant:\n",
    "                mrr_score += 1 / (rank + 1)\n",
    "                break\n",
    "    mrr = mrr_score / len(relevance_total)\n",
    "    \n",
    "    # Calculate precision and recall for each query\n",
    "    precision_scores = []\n",
    "    recall_scores = []\n",
    "    \n",
    "    for relevance_list in relevance_total:\n",
    "        # Precision\n",
    "        precision = sum(relevance_list) / len(relevance_list) if relevance_list else 0.0\n",
    "        precision_scores.append(precision)\n",
    "        \n",
    "        # Recall (assuming 1 relevant document per query)\n",
    "        recall = sum(relevance_list) / 1 if sum(relevance_list) > 0 else 0.0\n",
    "        recall_scores.append(recall)\n",
    "    \n",
    "    # Calculate averages\n",
    "    avg_precision = np.mean(precision_scores)\n",
    "    avg_recall = np.mean(recall_scores)\n",
    "    \n",
    "    # Calculate F1 score\n",
    "    f1_score = (2 * avg_precision * avg_recall) / (avg_precision + avg_recall) if (avg_precision + avg_recall) > 0 else 0.0\n",
    "    \n",
    "    return {\n",
    "        'hit_rate': hit_rate,\n",
    "        'mrr': mrr,\n",
    "        'precision': avg_precision,\n",
    "        'recall': avg_recall,\n",
    "    }\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_with_params(index, query: str, params: Dict[str, Any]) -> List[Dict]:\n",
    "    \"\"\"Perform search with specific parameters\"\"\"\n",
    "    results = index.search(\n",
    "        query=query,\n",
    "        filter_dict={},\n",
    "        boost_dict=params.get('boost', {}),\n",
    "        num_results=params.get('num_results', 10)\n",
    "    )\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_params(index, ground_truth: List[Dict], params: Dict[str, Any]) -> Dict[str, float]:\n",
    "    \"\"\"Evaluate a single parameter configuration\"\"\"\n",
    "    relevance_total = []\n",
    "    \n",
    "    for q in tqdm(ground_truth, desc=\"Evaluating configuration\"):\n",
    "        doc_id = q['id']\n",
    "        results = search_with_params(index, q['question'], params)\n",
    "        \n",
    "        # Check relevance of results\n",
    "        relevance = [doc['question_id'] == doc_id for doc in results]\n",
    "        relevance_total.append(relevance)\n",
    "    \n",
    "    return calculate_metrics(relevance_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_param_combinations(param_grid: Dict[str, List[Any]]) -> List[Dict[str, Any]]:\n",
    "    \"\"\"Generate all possible combinations of parameters\"\"\"\n",
    "    keys = param_grid.keys()\n",
    "    values = product(*param_grid.values())\n",
    "    return [dict(zip(keys, v)) for v in values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_params(index, ground_truth: List[Dict], param_grid: Dict[str, List[Any]]) -> Dict:\n",
    "    \"\"\"Find the best parameters through grid search\"\"\"\n",
    "    \n",
    "    # Generate all parameter combinations\n",
    "    param_combinations = generate_param_combinations(param_grid)\n",
    "    \n",
    "    # Store all results\n",
    "    all_results = []\n",
    "    best_score = -float('inf')\n",
    "    best_params = None\n",
    "    best_metrics = None\n",
    "    \n",
    "    # Test each parameter combination\n",
    "    for params in tqdm(param_combinations, desc=\"Testing parameter combinations\"):\n",
    "        metrics = evaluate_params(index, ground_truth, params)\n",
    "        \n",
    "        # Calculate overall score (average of all metrics)\n",
    "        score = np.mean([\n",
    "            metrics['hit_rate'],\n",
    "            metrics['mrr']\n",
    "        ])\n",
    "        \n",
    "        result = {\n",
    "            'params': params,\n",
    "            'metrics': metrics,\n",
    "            'overall_score': score\n",
    "        }\n",
    "        all_results.append(result)\n",
    "        \n",
    "        # Update best results if current score is better\n",
    "        if score > best_score:\n",
    "            best_score = score\n",
    "            best_params = params\n",
    "            best_metrics = metrics\n",
    "    \n",
    "    return {\n",
    "        'best_params': best_params,\n",
    "        'best_metrics': best_metrics,\n",
    "        'all_results': all_results\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example parameter grid\n",
    "default_param_grid = {\n",
    "    'boost': [\n",
    "        {'questions': 1.0, 'answers': 0.5},\n",
    "        {'questions': 2.0, 'answers': 1.0},\n",
    "        {'questions': 1.0, 'answers': 1.0},\n",
    "        {'questions': 3.0, 'answers': 0.5},\n",
    "    ],\n",
    "    'num_results': [5, 10, 15, 20]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6e504832cd741769e878efba4afe3c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing parameter combinations:   0%|          | 0/16 [00:01<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8163a4f3cac4cbcb755ee27a971eeca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09ecba28f2154810aa2fa005123cda5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2745c31e22e4d2eb0663541837f8b7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ca4f884eecb4de39c76b96b9f3ac93d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d07b21b19e149efaadf7b5bbc7e9bfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef7ccb7b76d343bfac61e39894a08328",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a5ff09ade8f4b59a444efc8733b2efe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f81e27e89c884d5da0b52718c92dfbf1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ab5616c6b31431e9499f0506de35f25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7b6065b88674511a79e9fa6bb40d431",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9700ff238cf04369b92590bc5b629237",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e214dfc711144ba396d74c5b0ff63a20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e499015811446db7fa4f41ce3be24b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b81c1ae0e641493c9a4da1843da74196",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccf3edaedc48415b8c64a59a1228636f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2065e0ce8e674634877840dadbaa0cec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating configuration:   0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'boost': {'questions': 1.0, 'answers': 1.0}, 'num_results': 20}\n",
      "\n",
      "Best Metrics:\n",
      "hit_rate: 0.9699\n",
      "mrr: 0.7146\n",
      "precision: 0.0485\n",
      "recall: 0.9699\n",
      "\n",
      "All configurations sorted by performance:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>boost</th>\n",
       "      <th>num_results</th>\n",
       "      <th>metric_hit_rate</th>\n",
       "      <th>metric_mrr</th>\n",
       "      <th>metric_precision</th>\n",
       "      <th>metric_recall</th>\n",
       "      <th>overall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>{'questions': 1.0, 'answers': 1.0}</td>\n",
       "      <td>20</td>\n",
       "      <td>0.969945</td>\n",
       "      <td>0.714573</td>\n",
       "      <td>0.048497</td>\n",
       "      <td>0.969945</td>\n",
       "      <td>0.842259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>{'questions': 1.0, 'answers': 1.0}</td>\n",
       "      <td>15</td>\n",
       "      <td>0.950820</td>\n",
       "      <td>0.713451</td>\n",
       "      <td>0.063388</td>\n",
       "      <td>0.950820</td>\n",
       "      <td>0.832135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>{'questions': 1.0, 'answers': 1.0}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.918033</td>\n",
       "      <td>0.710774</td>\n",
       "      <td>0.091803</td>\n",
       "      <td>0.918033</td>\n",
       "      <td>0.814403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>{'questions': 1.0, 'answers': 1.0}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.702368</td>\n",
       "      <td>0.171038</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.778780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'questions': 1.0, 'answers': 0.5}</td>\n",
       "      <td>20</td>\n",
       "      <td>0.912568</td>\n",
       "      <td>0.602561</td>\n",
       "      <td>0.045628</td>\n",
       "      <td>0.912568</td>\n",
       "      <td>0.757565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>{'questions': 2.0, 'answers': 1.0}</td>\n",
       "      <td>20</td>\n",
       "      <td>0.912568</td>\n",
       "      <td>0.602561</td>\n",
       "      <td>0.045628</td>\n",
       "      <td>0.912568</td>\n",
       "      <td>0.757565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'questions': 1.0, 'answers': 0.5}</td>\n",
       "      <td>15</td>\n",
       "      <td>0.890710</td>\n",
       "      <td>0.601329</td>\n",
       "      <td>0.059381</td>\n",
       "      <td>0.890710</td>\n",
       "      <td>0.746020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>{'questions': 2.0, 'answers': 1.0}</td>\n",
       "      <td>15</td>\n",
       "      <td>0.890710</td>\n",
       "      <td>0.601329</td>\n",
       "      <td>0.059381</td>\n",
       "      <td>0.890710</td>\n",
       "      <td>0.746020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'questions': 1.0, 'answers': 0.5}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.598455</td>\n",
       "      <td>0.085519</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.726823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>{'questions': 2.0, 'answers': 1.0}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.598455</td>\n",
       "      <td>0.085519</td>\n",
       "      <td>0.855191</td>\n",
       "      <td>0.726823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>{'questions': 3.0, 'answers': 0.5}</td>\n",
       "      <td>20</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.532316</td>\n",
       "      <td>0.042623</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.692388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>{'questions': 3.0, 'answers': 0.5}</td>\n",
       "      <td>15</td>\n",
       "      <td>0.814208</td>\n",
       "      <td>0.530218</td>\n",
       "      <td>0.054281</td>\n",
       "      <td>0.814208</td>\n",
       "      <td>0.672213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'questions': 1.0, 'answers': 0.5}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.756831</td>\n",
       "      <td>0.584791</td>\n",
       "      <td>0.151366</td>\n",
       "      <td>0.756831</td>\n",
       "      <td>0.670811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'questions': 2.0, 'answers': 1.0}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.756831</td>\n",
       "      <td>0.584791</td>\n",
       "      <td>0.151366</td>\n",
       "      <td>0.756831</td>\n",
       "      <td>0.670811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>{'questions': 3.0, 'answers': 0.5}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.751366</td>\n",
       "      <td>0.525145</td>\n",
       "      <td>0.075137</td>\n",
       "      <td>0.751366</td>\n",
       "      <td>0.638256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>{'questions': 3.0, 'answers': 0.5}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.661202</td>\n",
       "      <td>0.513661</td>\n",
       "      <td>0.132240</td>\n",
       "      <td>0.661202</td>\n",
       "      <td>0.587432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 boost  num_results  metric_hit_rate  \\\n",
       "11  {'questions': 1.0, 'answers': 1.0}           20         0.969945   \n",
       "10  {'questions': 1.0, 'answers': 1.0}           15         0.950820   \n",
       "9   {'questions': 1.0, 'answers': 1.0}           10         0.918033   \n",
       "8   {'questions': 1.0, 'answers': 1.0}            5         0.855191   \n",
       "3   {'questions': 1.0, 'answers': 0.5}           20         0.912568   \n",
       "7   {'questions': 2.0, 'answers': 1.0}           20         0.912568   \n",
       "2   {'questions': 1.0, 'answers': 0.5}           15         0.890710   \n",
       "6   {'questions': 2.0, 'answers': 1.0}           15         0.890710   \n",
       "1   {'questions': 1.0, 'answers': 0.5}           10         0.855191   \n",
       "5   {'questions': 2.0, 'answers': 1.0}           10         0.855191   \n",
       "15  {'questions': 3.0, 'answers': 0.5}           20         0.852459   \n",
       "14  {'questions': 3.0, 'answers': 0.5}           15         0.814208   \n",
       "0   {'questions': 1.0, 'answers': 0.5}            5         0.756831   \n",
       "4   {'questions': 2.0, 'answers': 1.0}            5         0.756831   \n",
       "13  {'questions': 3.0, 'answers': 0.5}           10         0.751366   \n",
       "12  {'questions': 3.0, 'answers': 0.5}            5         0.661202   \n",
       "\n",
       "    metric_mrr  metric_precision  metric_recall  overall_score  \n",
       "11    0.714573          0.048497       0.969945       0.842259  \n",
       "10    0.713451          0.063388       0.950820       0.832135  \n",
       "9     0.710774          0.091803       0.918033       0.814403  \n",
       "8     0.702368          0.171038       0.855191       0.778780  \n",
       "3     0.602561          0.045628       0.912568       0.757565  \n",
       "7     0.602561          0.045628       0.912568       0.757565  \n",
       "2     0.601329          0.059381       0.890710       0.746020  \n",
       "6     0.601329          0.059381       0.890710       0.746020  \n",
       "1     0.598455          0.085519       0.855191       0.726823  \n",
       "5     0.598455          0.085519       0.855191       0.726823  \n",
       "15    0.532316          0.042623       0.852459       0.692388  \n",
       "14    0.530218          0.054281       0.814208       0.672213  \n",
       "0     0.584791          0.151366       0.756831       0.670811  \n",
       "4     0.584791          0.151366       0.756831       0.670811  \n",
       "13    0.525145          0.075137       0.751366       0.638256  \n",
       "12    0.513661          0.132240       0.661202       0.587432  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# Run the optimization\n",
    "results = find_best_params(index, ground_truth, default_param_grid)\n",
    "\n",
    "# Print best parameters\n",
    "print(\"Best Parameters:\", results['best_params'])\n",
    "print(\"\\nBest Metrics:\")\n",
    "for metric, value in results['best_metrics'].items():\n",
    "    print(f\"{metric}: {value:.4f}\")\n",
    "\n",
    "# Create DataFrame of all results for analysis\n",
    "results_df = pd.DataFrame([\n",
    "    {\n",
    "        **r['params'],\n",
    "        **{f\"metric_{k}\": v for k, v in r['metrics'].items()},\n",
    "        'overall_score': r['overall_score']\n",
    "    }\n",
    "    for r in results['all_results']\n",
    "])\n",
    "\n",
    "# Display results sorted by overall score\n",
    "print(\"\\nAll configurations sorted by performance:\")\n",
    "display(results_df.sort_values('overall_score', ascending=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rag Flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_template=\"\"\"\n",
    "You are an expert mental health assistant specialized in providing detailed and accurate answers based on the given context.\n",
    "Answer the QUESTION based on the CONTEXT from our meantal health database.\n",
    "Use only the facts from the CONTEXT when answering the QUESTION.\n",
    "\n",
    "Here is the context:\n",
    "\n",
    "Context: {context}\n",
    "\n",
    "Please answer the following question based on the provided context:\n",
    "\n",
    "Question: {question}\n",
    "\n",
    "Provide a detailed and informative response. Ensure that your answer is clear, concise, and directly addresses the question while being relevant to the context provided.\n",
    "\n",
    "Your response should be in plain text and should not include any code blocks or extra formatting.\n",
    "\n",
    "Answer:\n",
    "\"\"\".strip()\n",
    "def build_prompt(query, search_results):\n",
    "    context = \"\"\n",
    "    \n",
    "    for doc in search_results:\n",
    "        context = context.format(**doc) + \"\\n\\n\"\n",
    "\n",
    "    prompt = prompt_template.format(question=query, context=context).strip()\n",
    "    return prompt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def llm(prompt,model):\n",
    "        response = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}])\n",
    "        return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rag(query, model='mixtral-8x7b-32768'):\n",
    "    search_results = search(query)\n",
    "    prompt = build_prompt(query, search_results)\n",
    "    #print(prompt)\n",
    "    answer = llm(prompt, model=model)\n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "query=\"What should I eat if I lost a friend\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I'm really sorry to hear that you're dealing with the loss of a friendship. It's important to remember that while food can't solve emotional pain, maintaining a balanced diet can help support your overall well-being during this difficult time.\\n\\nUnfortunately, the context provided doesn't give specific dietary recommendations for dealing with lost friendships. However, generally, it's recommended to eat a variety of nutrient-dense foods. This can include:\\n\\n1. Fruits and vegetables: These are high in vitamins, minerals, and fiber, which can support your physical health.\\n2. Lean proteins: Foods like chicken, turkey, fish, eggs, and tofu can help maintain muscle mass and support your body's healing processes.\\n3. Whole grains: Foods like brown rice, whole wheat bread, and quinoa can provide energy and help keep you feeling full.\\n4. Healthy fats: Foods like avocados, nuts, seeds, and olive oil can support brain health and help keep you feeling satisfied.\\n\\nRemember, it's also important to stay hydrated and to listen to your body's hunger and fullness cues. If you're finding it hard to eat, try smaller, more frequent meals. If you're turning to food for comfort, try to be mindful of your eating habits and consider healthier alternatives or seeking support from a mental health professional.\""
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rag(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rag Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt2_template = \"\"\"\n",
    "You are an expert evaluator for a RAG system.\n",
    "Your task is to analyze the relevance of the generated answer to the given question.\n",
    "Based on the relevance of the generated answer, you will classify it\n",
    "as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
    "\n",
    "Here is the data for evaluation:\n",
    "\n",
    "Question: {question}\n",
    "Generated Answer: {answer_llm}\n",
    "\n",
    "Please analyze the content and context of the generated answer in relation to the question\n",
    "and provide your evaluation in parsable JSON without using code blocks:\n",
    "\n",
    "{{\n",
    "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
    "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
    "}}\n",
    "\"\"\".strip()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "366"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 1590140,\n",
       " 'question': \"How do mental illnesses affect a person's daily functioning and relationships?\"}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "record=ground_truth[0]\n",
    "question=record[\"question\"]\n",
    "answer_llm=rag(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert evaluator for a RAG system.\n",
      "Your task is to analyze the relevance of the generated answer to the given question.\n",
      "Based on the relevance of the generated answer, you will classify it\n",
      "as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
      "\n",
      "Here is the data for evaluation:\n",
      "\n",
      "Question: How do mental illnesses affect a person's daily functioning and relationships?\n",
      "Generated Answer: Mental illnesses can significantly impact a person's daily functioning and relationships in various ways. According to the context provided, mental illnesses can affect a person's ability to perform routine activities, maintain employment, manage finances, and handle responsibilities at home. They may also experience difficulty in concentrating, making decisions, and regulating emotions, which can further interfere with their daily life.\n",
      "\n",
      "Furthermore, mental illnesses can strain a person's relationships with family, friends, and colleagues. The symptoms of mental illnesses, such as irritability, mood swings, and social withdrawal, can cause misunderstandings, conflicts, and tension in interpersonal relationships. Mental illnesses can also affect a person's communication, empathy, and trust, making it challenging for them to build and maintain meaningful connections with others.\n",
      "\n",
      "In some cases, mental illnesses can lead to social isolation, loneliness, and discrimination, further exacerbating the negative impact on a person's daily functioning and relationships. It is essential to provide support, understanding, and accommodations to individuals experiencing mental health issues to help them manage their symptoms and improve their quality of life.\n",
      "\n",
      "Please analyze the content and context of the generated answer in relation to the question\n",
      "and provide your evaluation in parsable JSON without using code blocks:\n",
      "\n",
      "{\n",
      "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
      "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "prompt=prompt2_template.format(question=question,answer_llm=answer_llm)\n",
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ff19c2ce6cc44199c08e01f7d9c4ab4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/366 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import json\n",
    "\n",
    "evaluations=[]\n",
    "\n",
    "for record in tqdm(ground_truth):\n",
    "    id=record['id']\n",
    "    if id in evaluations:\n",
    "        continue\n",
    "    question=record['question']\n",
    "    answer_llm=rag(question)\n",
    "    prompt=prompt2_template.format(\n",
    "        question=question,\n",
    "        answer_llm=answer_llm)\n",
    "    evaluation=llm(prompt,model='mixtral-8x7b-32768')\n",
    "    evaluation = json.loads(evaluation)\n",
    "    evaluations.append((record,answer_llm,evaluation))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval = pd.DataFrame(evaluations, columns=['record', 'answer', 'evaluation'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record</th>\n",
       "      <th>answer</th>\n",
       "      <th>evaluation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'id': 1590140, 'question': 'How do mental ill...</td>\n",
       "      <td>Mental illnesses can significantly impact a pe...</td>\n",
       "      <td>{'Relevance': 'RELEVANT', 'Explanation': 'The ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'id': 1590140, 'question': 'What are some exa...</td>\n",
       "      <td>Sure, I'd be happy to help answer your questio...</td>\n",
       "      <td>{'Relevance': 'RELEVANT', 'Explanation': 'The ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'id': 1590140, 'question': 'Why is it a misco...</td>\n",
       "      <td>Mental illness is not a reflection of a person...</td>\n",
       "      <td>{'Relevance': 'RELEVANT', 'Explanation': 'The ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'id': 1590140, 'question': 'How are mental il...</td>\n",
       "      <td>Mental illnesses are treated using a variety o...</td>\n",
       "      <td>{'Relevance': 'RELEVANT', 'Explanation': 'The ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'id': 1590140, 'question': 'Can a person with...</td>\n",
       "      <td>Yes, a person with a mental illness can become...</td>\n",
       "      <td>{'Relevance': 'RELEVANT', 'Explanation': 'The ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              record  \\\n",
       "0  {'id': 1590140, 'question': 'How do mental ill...   \n",
       "1  {'id': 1590140, 'question': 'What are some exa...   \n",
       "2  {'id': 1590140, 'question': 'Why is it a misco...   \n",
       "3  {'id': 1590140, 'question': 'How are mental il...   \n",
       "4  {'id': 1590140, 'question': 'Can a person with...   \n",
       "\n",
       "                                              answer  \\\n",
       "0  Mental illnesses can significantly impact a pe...   \n",
       "1  Sure, I'd be happy to help answer your questio...   \n",
       "2  Mental illness is not a reflection of a person...   \n",
       "3  Mental illnesses are treated using a variety o...   \n",
       "4  Yes, a person with a mental illness can become...   \n",
       "\n",
       "                                          evaluation  \n",
       "0  {'Relevance': 'RELEVANT', 'Explanation': 'The ...  \n",
       "1  {'Relevance': 'RELEVANT', 'Explanation': 'The ...  \n",
       "2  {'Relevance': 'RELEVANT', 'Explanation': 'The ...  \n",
       "3  {'Relevance': 'RELEVANT', 'Explanation': 'The ...  \n",
       "4  {'Relevance': 'RELEVANT', 'Explanation': 'The ...  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval = pd.DataFrame(evaluations, columns=['record', 'answer', 'evaluation'])\n",
    "\n",
    "df_eval['id'] = df_eval.record.apply(lambda d: d['id'])\n",
    "df_eval['question'] = df_eval.record.apply(lambda d: d['question'])\n",
    "\n",
    "df_eval['relevance'] = df_eval.evaluation.apply(lambda d: d['Relevance'])\n",
    "df_eval['explanation'] = df_eval.evaluation.apply(lambda d: d['Explanation'])\n",
    "\n",
    "del df_eval['record']\n",
    "del df_eval['evaluation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>relevance</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mental illnesses can significantly impact a pe...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>How do mental illnesses affect a person's dail...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer is highly relevant to the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sure, I'd be happy to help answer your questio...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>What are some examples of serious mental illne...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer provides a detailed list ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mental illness is not a reflection of a person...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>Why is it a misconception to associate mental ...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mental illnesses are treated using a variety o...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>How are mental illnesses treated, and what is ...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer fully addresses the quest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yes, a person with a mental illness can become...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>Can a person with mental illness become indepe...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer       id  \\\n",
       "0  Mental illnesses can significantly impact a pe...  1590140   \n",
       "1  Sure, I'd be happy to help answer your questio...  1590140   \n",
       "2  Mental illness is not a reflection of a person...  1590140   \n",
       "3  Mental illnesses are treated using a variety o...  1590140   \n",
       "4  Yes, a person with a mental illness can become...  1590140   \n",
       "\n",
       "                                            question relevance  \\\n",
       "0  How do mental illnesses affect a person's dail...  RELEVANT   \n",
       "1  What are some examples of serious mental illne...  RELEVANT   \n",
       "2  Why is it a misconception to associate mental ...  RELEVANT   \n",
       "3  How are mental illnesses treated, and what is ...  RELEVANT   \n",
       "4  Can a person with mental illness become indepe...  RELEVANT   \n",
       "\n",
       "                                         explanation  \n",
       "0  The generated answer is highly relevant to the...  \n",
       "1  The generated answer provides a detailed list ...  \n",
       "2  The generated answer directly addresses the qu...  \n",
       "3  The generated answer fully addresses the quest...  \n",
       "4  The generated answer directly addresses the qu...  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relevance\n",
      "RELEVANT           348\n",
      "PARTLY_RELEVANT     17\n",
      "NON_RELEVANT         1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Count the occurrences of each relevance category\n",
    "relevance_counts = df_eval['relevance'].value_counts()\n",
    "print(relevance_counts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing seaborn..."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading .env environment variables...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Resolving seaborn...\n",
      "[    ] Installing...\n",
      "[=   ] Installing seaborn...\n",
      "Installation Succeeded\n",
      "[==  ] Installing seaborn...\n",
      "[==  ] Installing seaborn...\n",
      "\n",
      "Installing dependencies from Pipfile.lock (7b2755)...\n"
     ]
    }
   ],
   "source": [
    "!pipenv install seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relevance\n",
      "RELEVANT           348\n",
      "PARTLY_RELEVANT     17\n",
      "NON_RELEVANT         1\n",
      "Name: count, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\AppData\\Local\\Temp\\ipykernel_16892\\1345086306.py:15: FutureWarning: \n",
      "\n",
      "Passing `palette` without assigning `hue` is deprecated and will be removed in v0.14.0. Assign the `x` variable to `hue` and set `legend=False` for the same effect.\n",
      "\n",
      "  ax = sns.countplot(data=df_eval, x='relevance', palette='viridis')\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArcAAAHWCAYAAABt3aEVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABghklEQVR4nO3de3yP9eP/8efOZmwzzIw1Z4YNCa1yiDHHIj7hI0YO0SaHQis5VVYph+SQYpOc4lMqhZxVJCmhnHMqRjlsjrPD6/eH395fb9uwGZurx/12u263vV/X67qu1+va+3rvuev9er/eDsYYIwAAAMACHPO6AQAAAEBuIdwCAADAMgi3AAAAsAzCLQAAACyDcAsAAADLINwCAADAMgi3AAAAsAzCLQAAACyDcAsAAADLINzC0kaNGiUHB4e7cqxGjRqpUaNGtsfr1q2Tg4ODFi9efFeO3717d5UpU+auHCunzp8/r169esnPz08ODg4aOHDgXT3+9b8j3Pvi4uLk4OCgQ4cO5XVT7OTXdl3r0KFDcnBw0Ntvv53XTQFyFeEW94z0PxbpS4ECBeTv76/w8HC9++67OnfuXK4c59ixYxo1apS2bduWK/vLTfm5bbdi7NixiouLU79+/TRnzhx17do1y7plypSx+317eHiobt26+uijj+5ii/8dGjVqZHeur112796d1827Y8aOHaslS5bkdTOUlpam6dOnq2bNmipUqJBKlCihFi1aaOPGjTna341+ny4uLrna9s8++0zh4eHy9/eXm5ubSpcurQ4dOmjnzp129U6dOqVx48apQYMGKl68uLy9vfXggw9q4cKFGfbZvXv3LNvv4OCgv/76y67+lStXNHbsWFWpUkUFChRQiRIl1KpVK/3555929fbt26dOnTqpdOnSKliwoKpUqaIxY8bo4sWLuXpOkPec87oBQHaNGTNGZcuWVXJysuLj47Vu3ToNHDhQ48eP1xdffKGQkBBb3eHDh+vFF1/M1v6PHTum0aNHq0yZMqpZs+Ytb/fNN99k6zg5caO2ffDBB0pLS7vjbbgda9as0YMPPqiRI0feUv2aNWvq+eeflyQdP35cH374oSIiIpSUlKTevXvfyab+65QuXVoxMTEZyv39/fOgNbeua9eu6tSpk9zc3LK97dixY9WhQwe1bds29xuWDUOGDNH48eP11FNP6dlnn9XZs2f1/vvvq2HDhvr+++9Vt27dbO3v5ZdfVq9evezKLly4oL59+6pZs2a52XTt2LFDRYoU0YABA1SsWDHFx8dr1qxZqlu3rjZt2qQaNWpIkjZt2qSXX35ZLVu21PDhw+Xs7Kz//e9/6tSpk37//XeNHj3ats9nnnlGYWFhdscxxqhv374qU6aMSpUqZStPTk5Wq1attHHjRvXu3VshISE6c+aMNm/erISEBJUuXVqSdPToUdWtW1deXl6KioqSj4+PNm3apJEjR2rr1q36/PPPc/W8II8Z4B4RGxtrJJktW7ZkWLd69Wrj7u5uAgMDzcWLF2/rOFu2bDGSTGxs7C3Vv3DhQqbla9euNZLMokWLbqs9t9O2/KZs2bKmVatWt1Q3MDAwQ92TJ0+aQoUKmaCgoBwdv2HDhqZhw4Y52tbKGjZsaKpVq3ZH9p2Wlnbb1+Sd4uHhYSIiIm6p7vnz57O17/TXq4MHD96wXnJysnF3dzcdOnSwK//jjz+MJPPcc89l67hZmTNnjpFk5s6days7ePCgkWTGjRuXK8dIFx8fb5ydnc0zzzxjK/vjjz/MoUOH7OqlpaWZxo0bGzc3t5ue32+//dZIMq+//rpd+ZtvvmlcXFzM5s2bb7j966+/biSZnTt32pV369bNSDKnT5++la7hHsGwBFhC48aN9corr+jw4cP6+OOPbeWZjblduXKlHnnkEXl7e6tQoUKqXLmyXnrpJUlXx8nWqVNHktSjRw/b22BxcXGSrr7dV716dW3dulUNGjRQwYIFbdtmNZ4zNTVVL730kvz8/OTh4aHHHntMR48etatTpkwZde/ePcO21+7zZm3LbMzthQsX9PzzzysgIEBubm6qXLmy3n77bRlj7Oo5ODgoKipKS5YsUfXq1eXm5qZq1app+fLlmZ/w65w8eVI9e/ZUiRIlVKBAAdWoUUOzZ8+2rU8ff3zw4EF99dVXtrZndzxi8eLFVaVKFR04cMCuPC0tTRMnTlS1atVsb0s+88wzOnPmzE33mZSUpJEjR6pChQpyc3NTQECAhg4dqqSkJFud6tWr69FHH82wbVpamkqVKqUOHTrYyt5++2099NBDKlq0qNzd3VW7du1Mx11n55z/9ddf6tmzp+2t37Jly6pfv366cuWKrc7Zs2c1cOBA2++6QoUKevPNN3Ptbn5KSopeffVVlS9fXm5ubipTpoxeeuklu/MkXX0ut27dWitWrNADDzwgd3d3vf/++1nuN/2a2r59uxo2bKiCBQuqQoUKtnO2fv161atXT+7u7qpcubJWrVplt/31Y1vXrFkjR0dHjRgxwq7evHnz5ODgoGnTpkm6ev4vXLig2bNn256P6ddg+uvG77//rv/+978qUqSIHnnkEUnS9u3b1b17d5UrV04FChSQn5+fnn76aZ06dSpH5zU5OVmXLl1SiRIl7Mp9fX3l6Ogod3d3SdKlS5dUpUoVValSRZcuXbLVO336tEqWLKmHHnpIqampWR5n3rx58vDw0OOPP57p+gkTJigwMFDu7u5q2LBhhmEF2eHr66uCBQvq7NmztrKyZcsqMDDQrp6Dg4Patm2rpKQk/fHHHzfcZ/rv77///a+tLC0tTZMmTVK7du1Ut25dpaSkZDnEIDExUZIynOeSJUvK0dFRrq6u2eki8ru8TtfArbrRnVtjjDl69KiRZHcHZOTIkebap/nOnTuNq6ureeCBB8ykSZPM9OnTzQsvvGAaNGhgjLl6x2HMmDFGkunTp4+ZM2eOmTNnjjlw4IAx5uodLj8/P1O8eHHTv39/8/7775slS5bY1l17VzD9zm1wcLAJCQkx48ePNy+++KIpUKCAqVSpkt3drMDAwEzvIF27z5u1LSIiwgQGBtq2Tb8r4uDgYHr16mXee+8906ZNGyPJDBw40O44kkyNGjVMyZIlzauvvmomTpxoypUrZwoWLGj++eefG/5eLl68aIKCgoyLi4sZNGiQeffdd039+vWNJDNx4kRb2+fMmWOKFStmatasaWv7je7WZHbnNjk52fj5+ZkSJUrYlffq1cs4Ozub3r17m+nTp5thw4YZDw8PU6dOHXPlypVMz6cxxqSmpppmzZqZggULmoEDB5r333/fREVFGWdnZ/P444/b6o0ZM8Y4Ojqa48eP2x13/fr1Ge7Oly5d2jz77LPmvffeM+PHjzd169Y1kszSpUtzdM7/+usv4+/vb2vj9OnTzSuvvGKCgoLMmTNnjDFX3z0ICQkxRYsWNS+99JKZPn266datm3FwcDADBgzI8hxfe16qVKli/v77b7vl3LlztjoRERG262vKlCm2O15t27a121dgYKCpUKGCKVKkiHnxxRfN9OnTzdq1a294bH9/fxMQEGCGDBliJk+ebKpWrWqcnJzMggULjJ+fnxk1apSZOHGiKVWqlPHy8jKJiYm27TO7QxoZGWmcnZ3N1q1bjTHGHDt2zPj4+JiwsDCTlpZmjLl6J9PNzc3Ur1/f9nzcuHGjMeb/XjeqVq1qHn/8cTN16lQzZcoUY4wxb7/9tqlfv74ZM2aMmTFjhhkwYIBxd3c3devWte07q3ZlpV69esbDw8N8/PHH5vDhw+bXX381HTp0MEWLFrVd38YY88MPPxgnJyczaNAgW1mnTp2Mu7u72bNnT5b7P3nypHF2djZdunSxK0+/cxscHGzKlClj3nzzTTN69Gjj4+NjihcvbuLj42/a9nRnzpwxJ0+eNNu3bzdPP/20kWRmzJhx0+1eeuklI8kcO3YsyzpXrlwxRYsWNQ8//LBd+Y4dO4wk89prr5nevXsbV1dXW3/WrFljV3fZsmVGknnsscfML7/8Yo4cOWIWLFhgPD09M7we4t5HuMU942bh1hhjvLy8TK1atWyPrw+3EyZMMJLM33//neU+bvTWf8OGDY0kM3369EzXZRZuS5UqZffH+JNPPjGSzKRJk2xltxJub9a268PtkiVLbC/81+rQoYNxcHAw+/fvt5VJMq6urnZlv/76q5FkJk+enOFY15o4caKRZD7++GNb2ZUrV0xoaKgpVKiQXd8zC6xZCQwMNM2aNbMFrR07dpiuXbsaSSYyMtJWL/3tymvfbjXGmOXLl2cov/58zpkzxzg6Oppvv/3Wbtvp06cbSeb77783xhizZ8+eTM/Fs88+awoVKmT3j8r1b8FfuXLFVK9e3TRu3Niu/FbPebdu3Yyjo2Omz/v0MPXqq68aDw8Ps3fvXrv1L774onFycjJHjhzJsO210p/X1y/pz8lt27YZSaZXr152273wwgtGkl2QCAwMNJLM8uXLb3jM6489b948W9nu3buNJOPo6Gh++OEHW/mKFSsyPP8zC5EXLlwwFSpUMNWqVTOXL182rVq1Mp6enubw4cN2x85qWEL660bnzp0zrMtsiMX8+fONJLNhw4Ybtisr+/btM/fff7/duS9XrpzZvXt3hrrR0dHG0dHRbNiwwSxatMjun8isTJ482UgyX3/9tV15erh1d3c3f/75p6188+bNRpJdiL6ZypUr29peqFAhM3z4cJOamnrDbU6dOmV8fX1N/fr1b1jvyy+/NJLM1KlT7co//fRTI8kULVrUVKxY0cTGxprY2FhTsWJF4+rqan799Ve7+q+++qpxd3e3O88vv/zyLfcR9w6GJcBSChUqdMNZE7y9vSVJn3/+eY7frnVzc1OPHj1uuX63bt1UuHBh2+MOHTqoZMmS+vrrr3N0/Fv19ddfy8nJSc8995xd+fPPPy9jjJYtW2ZXHhYWpvLly9seh4SEyNPT86ZvF3799dfy8/NT586dbWUuLi567rnndP78ea1fvz7Hffjmm29UvHhxFS9eXMHBwZozZ4569OihcePG2eosWrRIXl5eatq0qf755x/bUrt2bRUqVEhr167Ncv+LFi1SUFCQqlSpYrdt48aNJcm2baVKlVSzZk27T3anpqZq8eLFatOmje2tY0l2P585c0YJCQmqX7++fv755wzHv9k5T0tL05IlS9SmTRs98MADGbZPH3KzaNEi1a9fX0WKFLHrR1hYmFJTU7Vhw4Ybn2hdHU6wcuVKu2Xo0KGSZHuuDh482G6b9A/7ffXVV3blZcuWVXh4+E2Pma5QoULq1KmT7XHlypXl7e2toKAg1atXz1ae/vPNnpMFCxZUXFycdu3apQYNGuirr77ShAkTdN99991ymySpb9++Gcqu/f1evnxZ//zzjx588EFJyvR3fCsKFy6satWqKTIyUp9++qmmTp2qlJQUtW3bVv/8849d3VGjRqlatWqKiIjQs88+q4YNG2a4xq83b948FS9eXE2bNs10fdu2be0+pFW3bl3Vq1cvW69RsbGxWr58uaZOnaqgoCBdunTphsMk0tLS1KVLF509e1aTJ0++aftdXFz05JNP2pWfP39eknTu3DmtXr1a3bt3V/fu3bVq1SoZY/TWW2/Z1S9TpowaNGigGTNm6H//+5+efvppjR07Vu+9994t9xP3BmZLgKWcP39evr6+Wa7v2LGjPvzwQ/Xq1UsvvviimjRpoieeeEIdOnSQo+Ot/a9XqlSpbI3Pqlixot1jBwcHVahQ4Y7Pf3n48GH5+/vbBWtJCgoKsq2/VmZ/+IsUKXLTcauHDx9WxYoVM5y/rI6THfXq1dNrr72m1NRU7dy5U6+99prOnDljd/737dunhISELH/vJ0+ezHL/+/bt065du1S8ePGbbtuxY0e99NJL+uuvv1SqVCmtW7dOJ0+eVMeOHe22Wbp0qV577TVt27bNbjxqZvMt3+yc//3330pMTFT16tWz7EN6P7Zv335L/ciKh4dHhk+opzt8+LAcHR1VoUIFu3I/Pz95e3tn+B2XLVv2pse7VunSpTOcHy8vLwUEBGQok3RLY6kffvhh9evXT1OmTFF4eLiefvrpbLVJyrwfp0+f1ujRo7VgwYIM5zUhISHLfSUkJNiNlXV1dZWPj49SUlIUFhamRo0a2YW8sLAwVatWTePGjdObb75pt92sWbNUp04dFShQQLGxsTecy/uPP/7Qpk2bFBUVJWfnzP/kX/8aJV39h+6TTz6RdHW87/V98/Pzs3scGhpq+7lTp0626z+rOXT79++v5cuX66OPPrLNqJCZ8+fP6/PPP1d4eLiKFi1qty79H42HH37Y7rly33336ZFHHrGbSm3BggXq06eP9u7da5tB4YknnlBaWpqGDRumzp07Z9g/7l2EW1jGn3/+qYSEhAx/gK/l7u6uDRs2aO3atfrqq6+0fPlyLVy4UI0bN9Y333wjJyenmx7n2js3uSWrP06pqam31KbckNVxzHUfPrubihUrZgtc4eHhqlKlilq3bq1JkybZ7iKmpaXJ19dXc+fOzXQfWQW+9G2Dg4M1fvz4TNdf+wezY8eOio6O1qJFizRw4EB98skn8vLyUvPmzW11vv32Wz322GNq0KCBpk6dqpIlS8rFxUWxsbGaN29ehv3n1jlPS0tT06ZNbXdar1epUqVs7S8rt/qFKNm9RrI6D7dzfpKSkrRu3TpJ0oEDB3Tx4kUVLFgwW+3KrB9PPvmkNm7cqCFDhtjmpU1LS1Pz5s1v+G7QgAED7D5k2bBhQ61bt04bNmzQzp07MzwHK1asqKCgIH3//fcZ9rVixQpJV+8c79u374b/TKQ/77p06XLjzt7AwoULM7xbdaPfQZEiRdS4cWPNnTs303A7evRoTZ06VW+88cYN57qWpCVLlujixYuZtj99mrrrPyQmXf1Q2y+//GJ7PHXqVNWqVcsWbNM99thjiouL0y+//JLlP3e49xBuYRlz5syRpJu+Hero6KgmTZqoSZMmGj9+vMaOHauXX35Za9euVVhYWK5/o9m+ffvsHhtjtH//frv5eIsUKWL3yeJ0hw8fVrly5WyPs9O2wMBArVq1SufOnbO7e5s+Kf/1n1zOqcDAQG3fvl1paWl2d29z+ziS1KpVKzVs2FBjx47VM888Iw8PD5UvX16rVq3Sww8/nO1QVb58ef36669q0qTJTc9t2bJlVbduXS1cuFBRUVH69NNP1bZtW7v5Vf/3v/+pQIECWrFihV15bGxs9jr6/xUvXlyenp43/eR6+fLldf78+Tv2xzkwMFBpaWnat2+f7Y6cJJ04cUJnz57N1d9xbhk5cqR27dqlt99+W8OGDdOLL76od999165Odq/1M2fOaPXq1Ro9erTdbAzXX+OZGTp0qJ566inb4yJFiki6eg4lZfoWfnJyslJSUuzKtm/frjFjxqhHjx7atm2bevXqpR07dtjual9v3rx5Kl++vG3oRGYya//evXtts6+Eh4dr5cqVN+7gdTK72ytJU6ZM0ahRozRw4EANGzbspvuZO3euChUqpMceeyzDuuDgYLm4uGT4Ugfp6pzg1/5je+LECds5v1ZycrIkZTjPuLcx5haWsGbNGr366qsqW7bsDe9QnD59OkNZ+pchpL+F7OHhIUmZhs2c+Oijj+zGAS9evFjHjx9XixYtbGXly5fXDz/8YDe109KlSzNMGZadtrVs2VKpqakZxpNNmDBBDg4Odse/HS1btlR8fLzdeNSUlBRNnjxZhQoVUsOGDXPlOOmGDRumU6dO6YMPPpB09U5aamqqXn311Qx1U1JSbniunnzySf3111+2fV3r0qVLunDhgl1Zx44d9cMPP2jWrFn6559/MgxJcHJykoODg11QOXToUI6/BcvR0VFt27bVl19+qZ9++inD+vS7Z08++aQ2bdpku6N3rbNnz972H+6WLVtKkiZOnGhXnn63sVWrVre1/9y2efNmvf322xo4cKCef/55DRkyRO+9916G8d8eHh7Zus7T7yRff9fy+vOSmapVqyosLMy21K5dW9L/3VVfsGCBXf2ff/5Ze/bsUa1atWxlycnJ6t69u/z9/TVp0iTFxcXpxIkTGjRoUKbH/OWXX7Rr1y676bMys2TJEruA+OOPP2rz5s2214iSJUvatf3af6IyG/Jy6NAhrV69OsM48YULF+q5555Tly5dsny35Fp///23Vq1apXbt2mV6171w4cJq2bKlNm7caPdNert27dLGjRvtxhhXqlRJv/zyi/bu3Wu3j/nz58vR0dHuZgPufdy5xT1n2bJl2r17t1JSUnTixAmtWbNGK1euVGBgoL744gsVKFAgy23HjBmjDRs2qFWrVgoMDNTJkyc1depUlS5d2jaPZfny5eXt7a3p06ercOHC8vDwUL169bI9jjCdj4+PHnnkEfXo0UMnTpzQxIkTVaFCBbtv2OrVq5cWL16s5s2b68knn9SBAwf08ccf233YKLtta9OmjR599FG9/PLLOnTokGrUqKFvvvlGn3/+uQYOHJhh3znVp08fvf/+++revbu2bt2qMmXKaPHixfr+++81ceLEDGN+b1eLFi1UvXp1jR8/XpGRkWrYsKGeeeYZxcTEaNu2bWrWrJlcXFy0b98+LVq0SJMmTbKbh/ZaXbt21SeffKK+fftq7dq1evjhh5Wamqrdu3frk08+sc3Vmu7JJ5/UCy+8oBdeeEE+Pj4Z7pS2atVK48ePV/PmzfXf//5XJ0+e1JQpU1ShQgVt3749R/0dO3asvvnmGzVs2FB9+vRRUFCQjh8/rkWLFum7776Tt7e3hgwZoi+++EKtW7dW9+7dVbt2bV24cEE7duzQ4sWLdejQIRUrVixHx5ekGjVqKCIiQjNmzNDZs2fVsGFD/fjjj5o9e7batm2b6RzAeeXy5cuKiIhQxYoV9frrr0u6+jb4l19+qR49emjHjh22fxJr166tVatWafz48fL391fZsmXtPsB2PU9PTzVo0EBvvfWWkpOTVapUKX3zzTc6ePBgjttbu3ZtNW3aVLNnz1ZiYqKaNWum48ePa/LkyXJ3d9fAgQNtddPHcq9evVqFCxdWSEiIRowYoeHDh6tDhw62f0LSpQ/VudmQhAoVKuiRRx5Rv379lJSUpIkTJ6po0aJZDnO5VnBwsJo0aaKaNWuqSJEi2rdvn2bOnKnk5GS98cYbtno//vijunXrpqJFi6pJkyYZhhE99NBDdu9SSVfDcEpKyg3bP3bsWK1evVqNGze2fbDu3XfflY+Pj20Ocunqt8AtW7ZM9evXV1RUlIoWLaqlS5dq2bJl6tWrV77/Jj5kU57N0wBkU/rUOumLq6ur8fPzM02bNjWTJk2ym3Iq3fVTga1evdo8/vjjxt/f37i6uhp/f3/TuXPnDFMoff7556Zq1arG2dnZbuqhG32TU1ZTgc2fP99ER0cbX19f4+7ublq1apVhSiJjjHnnnXdMqVKljJubm3n44YfNTz/9lOk3amXVtuunAjPGmHPnzplBgwYZf39/4+LiYipWrGjGjRtnNx+nMSbD9Frpspqi7HonTpwwPXr0MMWKFTOurq4mODg40+nKsjsVWFZ14+LiMkwJNWPGDFO7dm3j7u5uChcubIKDg83QoUPt5s/M7HxeuXLFvPnmm6ZatWrGzc3NFClSxNSuXduMHj3aJCQkZDj2ww8/nOm0WOlmzpxpKlasaNzc3EyVKlVMbGxshuehMdk754cPHzbdunUzxYsXN25ubqZcuXImMjLSJCUl2eqcO3fOREdHmwoVKhhXV1dTrFgx89BDD5m3337bbq7fzNzKN5QlJyeb0aNHm7JlyxoXFxcTEBBgoqOjzeXLlzO0/1Z/xzc6dlb7uf68XT/l1qBBg4yTk1OGb6z66aefjLOzs+nXr5+tbPfu3aZBgwa26aHSz3v67yuzKQP//PNP065dO+Pt7W28vLzMf/7zH3Ps2DEjyYwcOTLLdt3IxYsXzZgxY0zVqlWNu7u78fLyMq1btza//PKLrc7WrVuNs7Oz6d+/v922KSkppk6dOsbf398277ExV+dwLlWqlLn//vuzPO6131D2zjvvmICAANvcv9dPo5WVkSNHmgceeMAUKVLEODs7G39/f9OpUyezfft2u3rXv35fv2T2evHggw8aX19fk5KScsM2bN261YSFhRkPDw9TuHBh8/jjj2d4TTfm6hRnLVq0MH5+fsbFxcVUqlTJvP766yY5OfmW+op7h4MxefhpEQAAACAXMeYWAAAAlkG4BQAAgGUQbgEAAGAZhFsAAABYBuEWAAAAlkG4BQAAgGXwJQ66+r3sx44dU+HChXP9q1cBAABw+4wxOnfunPz9/e2+7v16hFtd/Q7qgICAvG4GAAAAbuLo0aMqXbp0lusJt5Lt60GPHj0qT0/PPG4NAAAArpeYmKiAgICbfq074VayDUXw9PQk3AIAAORjNxtCygfKAAAAYBmEWwAAAFgG4RYAAACWQbhFrjt37pwGDhyowMBAubu766GHHtKWLVts67t37y4HBwe7pXnz5jfc54YNG9SmTRv5+/vLwcFBS5YsyVDn7bfflq+vr3x9ffXOO+/Yrdu8ebNq166tlJSUXOkjAADIn/hAGXJdr169tHPnTs2ZM0f+/v76+OOPFRYWpt9//12lSpWSJDVv3lyxsbG2bdzc3G64zwsXLqhGjRp6+umn9cQTT2RYv337do0YMUJLly6VMUatW7dWs2bNFBwcrJSUFPXt21czZsyQszNPeQAArIy/9MhVly5d0v/+9z99/vnnatCggSRp1KhR+vLLLzVt2jS99tprkq6GWT8/v1veb4sWLdSiRYss1+/evVshISFq3LixJCkkJES7d+9WcHCwxo0bpwYNGqhOnTq30TMAAHAvINwiV6WkpCg1NVUFChSwK3d3d9d3331ne7xu3Tr5+vqqSJEiaty4sV577TUVLVo0x8cNDg7W3r17deTIERljtHfvXlWvXl0HDhxQbGystm7dmuN9AwCAewdjbpGrChcurNDQUL366qs6duyYUlNT9fHHH2vTpk06fvy4pKtDEj766COtXr1ab775ptavX68WLVooNTU1x8cNCgrS2LFj1bRpUzVr1kwxMTEKCgrSM888o7feeksrVqxQ9erVVatWLW3YsCG3ugsAAPIZ7twi182ZM0dPP/20SpUqJScnJ91///3q3Lmz7e5pp06dbHWDg4MVEhKi8uXLa926dWrSpEmOj9u3b1/17dvX9nj27Nm2sF25cmVt2bJFf/75pzp16qSDBw/edJwvAAC49+Tpndtp06YpJCTE9s1goaGhWrZsmW19o0aNMnyq/trwIklHjhxRq1atVLBgQfn6+mrIkCF8Ij6PlS9fXuvXr9f58+d19OhR/fjjj0pOTla5cuUyrV+uXDkVK1ZM+/fvz7U2/PPPPxo9erQmT56szZs3q1KlSqpYsaIeffRRJScna+/evbl2LAAAkH/k6Z3b0qVL64033lDFihVljNHs2bP1+OOP65dfflG1atUkSb1799aYMWNs2xQsWND2c2pqqlq1aiU/Pz9t3LhRx48fV7du3eTi4qKxY8fe9f7AnoeHhzw8PHTmzBmtWLFCb731Vqb1/vzzT506dUolS5bMtWMPGjRIgwYNUunSpbVlyxYlJyfb1qWPCwYAANaTp+G2TZs2do9ff/11TZs2TT/88IMt3BYsWDDLT9V/8803+v3337Vq1SqVKFFCNWvW1Kuvvqphw4Zp1KhRcnV1veN9QEYrVqyQMUaVK1fW/v37NWTIEFWpUkU9evTQ+fPnNXr0aLVv315+fn46cOCAhg4dqgoVKig8PNy2jyZNmqhdu3aKioqSJJ0/f97uzu7Bgwe1bds2+fj46L777rM7/sqVK7V3717Nnj1bklSnTh3t3r1by5Yt09GjR+Xk5KTKlSvfhTMBAADutnzzgbLU1FQtWLBAFy5cUGhoqK187ty5KlasmKpXr67o6GhdvHjRtm7Tpk0KDg5WiRIlbGXh4eFKTEzUb7/9luWxkpKSlJiYaLcg9yQkJCgyMlJVqlRRt27d9Mgjj2jFihVycXGRk5OTtm/frscee0yVKlVSz549Vbt2bX377bd2Y2APHDigf/75x/b4p59+Uq1atVSrVi1J0uDBg1WrVi2NGDHC7tiXLl1SVFSU3n//fTk6Xn16ly5dWpMnT1aPHj30+uuva/bs2XJ3d78LZwIAANxtDsYYk5cN2LFjh0JDQ3X58mUVKlRI8+bNU8uWLSVJM2bMUGBgoPz9/bV9+3YNGzZMdevW1aeffipJ6tOnjw4fPqwVK1bY9nfx4kV5eHjo66+/znJe1FGjRmn06NEZyhMSEuTp6XkHegkAAIDbkZiYKC8vr5vmtTyfLaFy5cratm2bEhIStHjxYkVERGj9+vWqWrWq+vTpY6sXHByskiVLqkmTJjpw4IDKly+f42NGR0dr8ODBtseJiYkKCAi4rX4AAAAg7+V5uHV1dVWFChUkSbVr19aWLVs0adIkvf/++xnq1qtXT5K0f/9+lS9fXn5+fvrxxx/t6pw4cUKSbvjtV25ubnd8GqhmHcfcvBJwl32zcMTNKwEAcA/LN2Nu06WlpSkpKSnTddu2bZMk26fqQ0NDtWPHDp08edJWZ+XKlfL09FTVqlXveFsBAACQv+Tpndvo6Gi1aNFC9913n86dO6d58+Zp3bp1WrFihQ4cOGAbf1u0aFFt375dgwYNUoMGDRQSEiJJatasmapWraquXbvqrbfeUnx8vIYPH67IyEgm6AcAAPgXytNwe/LkSXXr1k3Hjx+Xl5eXQkJCtGLFCjVt2lRHjx7VqlWrNHHiRF24cEEBAQFq3769hg8fbtveyclJS5cuVb9+/RQaGioPDw9FRETYzYsLAACAf488DbczZ87Mcl1AQIDWr19/030EBgbq66+/zs1mAQAA4B6V78bcAgAAADlFuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBl5Gm4nTZtmkJCQuTp6SlPT0+FhoZq2bJltvWXL19WZGSkihYtqkKFCql9+/Y6ceKE3T6OHDmiVq1aqWDBgvL19dWQIUOUkpJyt7sCAACAfCBPw23p0qX1xhtvaOvWrfrpp5/UuHFjPf744/rtt98kSYMGDdKXX36pRYsWaf369Tp27JieeOIJ2/apqalq1aqVrly5oo0bN2r27NmKi4vTiBEj8qpLAAAAyEMOxhiT1424lo+Pj8aNG6cOHTqoePHimjdvnjp06CBJ2r17t4KCgrRp0yY9+OCDWrZsmVq3bq1jx46pRIkSkqTp06dr2LBh+vvvv+Xq6npLx0xMTJSXl5cSEhLk6emZK/1o1nFMruwHyE3fLOQfPwDAvelW81q+GXObmpqqBQsW6MKFCwoNDdXWrVuVnJyssLAwW50qVarovvvu06ZNmyRJmzZtUnBwsC3YSlJ4eLgSExNtd38zk5SUpMTERLsFAAAA9748D7c7duxQoUKF5Obmpr59++qzzz5T1apVFR8fL1dXV3l7e9vVL1GihOLj4yVJ8fHxdsE2fX36uqzExMTIy8vLtgQEBORupwAAAJAn8jzcVq5cWdu2bdPmzZvVr18/RURE6Pfff7+jx4yOjlZCQoJtOXr06B09HgAAAO4O57xugKurqypUqCBJql27trZs2aJJkyapY8eOunLlis6ePWt39/bEiRPy8/OTJPn5+enHH3+021/6bArpdTLj5uYmNze3XO4JAAAA8lqe37m9XlpampKSklS7dm25uLho9erVtnV79uzRkSNHFBoaKkkKDQ3Vjh07dPLkSVudlStXytPTU1WrVr3rbQcAAEDeytM7t9HR0WrRooXuu+8+nTt3TvPmzdO6deu0YsUKeXl5qWfPnho8eLB8fHzk6emp/v37KzQ0VA8++KAkqVmzZqpataq6du2qt956S/Hx8Ro+fLgiIyO5MwsAAPAvlKfh9uTJk+rWrZuOHz8uLy8vhYSEaMWKFWratKkkacKECXJ0dFT79u2VlJSk8PBwTZ061ba9k5OTli5dqn79+ik0NFQeHh6KiIjQmDFMwwUAAPBvlO/muc0LzHOLfwvmuQUA3KvuuXluAQAAgNtFuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJaRp+E2JiZGderUUeHCheXr66u2bdtqz549dnUaNWokBwcHu6Vv3752dY4cOaJWrVqpYMGC8vX11ZAhQ5SSknI3uwIAAIB8wDkvD75+/XpFRkaqTp06SklJ0UsvvaRmzZrp999/l4eHh61e7969NWbMGNvjggUL2n5OTU1Vq1at5Ofnp40bN+r48ePq1q2bXFxcNHbs2LvaHwAAAOStPA23y5cvt3scFxcnX19fbd26VQ0aNLCVFyxYUH5+fpnu45tvvtHvv/+uVatWqUSJEqpZs6ZeffVVDRs2TKNGjZKrq+sd7QMAAADyj3w15jYhIUGS5OPjY1c+d+5cFStWTNWrV1d0dLQuXrxoW7dp0yYFBwerRIkStrLw8HAlJibqt99+y/Q4SUlJSkxMtFsAAABw78vTO7fXSktL08CBA/Xwww+revXqtvL//ve/CgwMlL+/v7Zv365hw4Zpz549+vTTTyVJ8fHxdsFWku1xfHx8pseKiYnR6NGj71BPAAAAkFfyTbiNjIzUzp079d1339mV9+nTx/ZzcHCwSpYsqSZNmujAgQMqX758jo4VHR2twYMH2x4nJiYqICAgZw0HAABAvpEvhiVERUVp6dKlWrt2rUqXLn3DuvXq1ZMk7d+/X5Lk5+enEydO2NVJf5zVOF03Nzd5enraLQAAALj35Wm4NcYoKipKn332mdasWaOyZcvedJtt27ZJkkqWLClJCg0N1Y4dO3Ty5ElbnZUrV8rT01NVq1a9I+0GAABA/pSnwxIiIyM1b948ff755ypcuLBtjKyXl5fc3d114MABzZs3Ty1btlTRokW1fft2DRo0SA0aNFBISIgkqVmzZqpataq6du2qt956S/Hx8Ro+fLgiIyPl5uaWl90DAADAXZand26nTZumhIQENWrUSCVLlrQtCxculCS5urpq1apVatasmapUqaLnn39e7du315dffmnbh5OTk5YuXSonJyeFhobqqaeeUrdu3ezmxQUAAMC/Q57euTXG3HB9QECA1q9ff9P9BAYG6uuvv86tZgEAAOAelS8+UAYAAADkBsItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALCNPw21MTIzq1KmjwoULy9fXV23bttWePXvs6ly+fFmRkZEqWrSoChUqpPbt2+vEiRN2dY4cOaJWrVqpYMGC8vX11ZAhQ5SSknI3uwIAAIB8IE/D7fr16xUZGakffvhBK1euVHJyspo1a6YLFy7Y6gwaNEhffvmlFi1apPXr1+vYsWN64oknbOtTU1PVqlUrXblyRRs3btTs2bMVFxenESNG5EWXAAAAkIccjDEmrxuR7u+//5avr6/Wr1+vBg0aKCEhQcWLF9e8efPUoUMHSdLu3bsVFBSkTZs26cEHH9SyZcvUunVrHTt2TCVKlJAkTZ8+XcOGDdPff/8tV1fXmx43MTFRXl5eSkhIkKenZ670pVnHMbmyHyA3fbOQf/oAAPemW81r+WrMbUJCgiTJx8dHkrR161YlJycrLCzMVqdKlSq67777tGnTJknSpk2bFBwcbAu2khQeHq7ExET99ttvmR4nKSlJiYmJdgsAAADufTkKt+XKldOpU6cylJ89e1blypXLUUPS0tI0cOBAPfzww6pevbokKT4+Xq6urvL29rarW6JECcXHx9vqXBts09enr8tMTEyMvLy8bEtAQECO2gwAAID8JUfh9tChQ0pNTc1QnpSUpL/++itHDYmMjNTOnTu1YMGCHG2fHdHR0UpISLAtR48evePHBAAAwJ3nnJ3KX3zxhe3nFStWyMvLy/Y4NTVVq1evVpkyZbLdiKioKC1dulQbNmxQ6dKlbeV+fn66cuWKzp49a3f39sSJE/Lz87PV+fHHH+32lz6bQnqd67m5ucnNzS3b7QQAAED+lq1w27ZtW0mSg4ODIiIi7Na5uLioTJkyeuedd255f8YY9e/fX5999pnWrVunsmXL2q2vXbu2XFxctHr1arVv316StGfPHh05ckShoaGSpNDQUL3++us6efKkfH19JUkrV66Up6enqlatmp3uAQAA4B6XrXCblpYmSSpbtqy2bNmiYsWK3dbBIyMjNW/ePH3++ecqXLiwbYysl5eX3N3d5eXlpZ49e2rw4MHy8fGRp6en+vfvr9DQUD344IOSpGbNmqlq1arq2rWr3nrrLcXHx2v48OGKjIzk7iwAAMC/TLbCbbqDBw/mysGnTZsmSWrUqJFdeWxsrLp37y5JmjBhghwdHdW+fXslJSUpPDxcU6dOtdV1cnLS0qVL1a9fP4WGhsrDw0MREREaM4apuAAAAP5tcjzP7erVq7V69WqdPHnSdkc33axZs3KlcXcL89zi34J5bgEA96pbzWs5unM7evRojRkzRg888IBKliwpBweHHDcUAAAAyC05CrfTp09XXFycunbtmtvtAQAAAHIsR/PcXrlyRQ899FButwUAAAC4LTkKt7169dK8efNyuy0AAADAbcnRsITLly9rxowZWrVqlUJCQuTi4mK3fvz48bnSOAAAACA7chRut2/frpo1a0qSdu7cabeOD5cBAAAgr+Qo3K5duza32wEAAADcthyNuQUAAADyoxzduX300UdvOPxgzZo1OW4QAAAAkFM5Crfp423TJScna9u2bdq5c6ciIiJyo10AAABAtuUo3E6YMCHT8lGjRun8+fO31SAAAAAgp3J1zO1TTz2lWbNm5eYuAQAAgFuWq+F206ZNKlCgQG7uEgAAALhlORqW8MQTT9g9Nsbo+PHj+umnn/TKK6/kSsMAAACA7MpRuPXy8rJ77OjoqMqVK2vMmDFq1qxZrjQMAAAAyK4chdvY2NjcbgcAAABw23IUbtNt3bpVu3btkiRVq1ZNtWrVypVGAQAAADmRo3B78uRJderUSevWrZO3t7ck6ezZs3r00Ue1YMECFS9ePDfbCAAAANySHM2W0L9/f507d06//fabTp8+rdOnT2vnzp1KTEzUc889l9ttBAAAAG5Jju7cLl++XKtWrVJQUJCtrGrVqpoyZQofKAMAAECeydGd27S0NLm4uGQod3FxUVpa2m03CgAAAMiJHIXbxo0ba8CAATp27Jit7K+//tKgQYPUpEmTXGscAAAAkB05CrfvvfeeEhMTVaZMGZUvX17ly5dX2bJllZiYqMmTJ+d2GwEAAIBbkqMxtwEBAfr555+1atUq7d69W5IUFBSksLCwXG0cAAAAkB3ZunO7Zs0aVa1aVYmJiXJwcFDTpk3Vv39/9e/fX3Xq1FG1atX07bff3qm2AgAAADeUrXA7ceJE9e7dW56enhnWeXl56ZlnntH48eNzrXEAAABAdmQr3P76669q3rx5luubNWumrVu33najAAAAgJzIVrg9ceJEplOApXN2dtbff/99240CAAAAciJb4bZUqVLauXNnluu3b9+ukiVL3najAAAAgJzIVrht2bKlXnnlFV2+fDnDukuXLmnkyJFq3bp1rjUOAAAAyI5sTQU2fPhwffrpp6pUqZKioqJUuXJlSdLu3bs1ZcoUpaam6uWXX74jDQUAAABuJlvhtkSJEtq4caP69eun6OhoGWMkSQ4ODgoPD9eUKVNUokSJO9JQAAAA4Gay/SUOgYGB+vrrr3XmzBnt379fxhhVrFhRRYoUuRPtAwAAAG5Zjr6hTJKKFCmiOnXq5GZbAAAAgNuSrQ+UAQAAAPkZ4RYAAACWQbgFAACAZRBuAQAAYBmEWwAAAFgG4RYAAACWQbgFAACAZRBuAQAAYBl5Gm43bNigNm3ayN/fXw4ODlqyZInd+u7du8vBwcFuad68uV2d06dPq0uXLvL09JS3t7d69uyp8+fP38VeAAAAIL/I03B74cIF1ahRQ1OmTMmyTvPmzXX8+HHbMn/+fLv1Xbp00W+//aaVK1dq6dKl2rBhg/r06XOnmw4AAIB8KMdfv5sbWrRooRYtWtywjpubm/z8/DJdt2vXLi1fvlxbtmzRAw88IEmaPHmyWrZsqbffflv+/v653mYAAADkX/l+zO26devk6+urypUrq1+/fjp16pRt3aZNm+Tt7W0LtpIUFhYmR0dHbd68Oct9JiUlKTEx0W4BAADAvS9fh9vmzZvro48+0urVq/Xmm29q/fr1atGihVJTUyVJ8fHx8vX1tdvG2dlZPj4+io+Pz3K/MTEx8vLysi0BAQF3tB8AAAC4O/J0WMLNdOrUyfZzcHCwQkJCVL58ea1bt05NmjTJ8X6jo6M1ePBg2+PExEQCLgAAgAXk6zu31ytXrpyKFSum/fv3S5L8/Px08uRJuzopKSk6ffp0luN0pavjeD09Pe0WAAAA3PvuqXD7559/6tSpUypZsqQkKTQ0VGfPntXWrVttddasWaO0tDTVq1cvr5oJAACAPJKnwxLOnz9vuwsrSQcPHtS2bdvk4+MjHx8fjR49Wu3bt5efn58OHDigoUOHqkKFCgoPD5ckBQUFqXnz5urdu7emT5+u5ORkRUVFqVOnTsyUAAAA8C+Up3duf/rpJ9WqVUu1atWSJA0ePFi1atXSiBEj5OTkpO3bt+uxxx5TpUqV1LNnT9WuXVvffvut3NzcbPuYO3euqlSpoiZNmqhly5Z65JFHNGPGjLzqEgAAAPJQnt65bdSokYwxWa5fsWLFTffh4+OjefPm5WazAAAAcI+6p8bcAgAAADdCuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJZBuAUAAIBlEG4BAABgGYRbAAAAWAbhFgAAAJaRp+F2w4YNatOmjfz9/eXg4KAlS5bYrTfGaMSIESpZsqTc3d0VFhamffv22dU5ffq0unTpIk9PT3l7e6tnz546f/78XewFAAAA8os8DbcXLlxQjRo1NGXKlEzXv/XWW3r33Xc1ffp0bd68WR4eHgoPD9fly5dtdbp06aLffvtNK1eu1NKlS7Vhwwb16dPnbnUBAAAA+YhzXh68RYsWatGiRabrjDGaOHGihg8frscff1yS9NFHH6lEiRJasmSJOnXqpF27dmn58uXasmWLHnjgAUnS5MmT1bJlS7399tvy9/e/a30BAABA3su3Y24PHjyo+Ph4hYWF2cq8vLxUr149bdq0SZK0adMmeXt724KtJIWFhcnR0VGbN2/Oct9JSUlKTEy0WwAAAHDvy7fhNj4+XpJUokQJu/ISJUrY1sXHx8vX19duvbOzs3x8fGx1MhMTEyMvLy/bEhAQkMutBwAAQF7It+H2ToqOjlZCQoJtOXr0aF43CQAAALkg34ZbPz8/SdKJEyfsyk+cOGFb5+fnp5MnT9qtT0lJ0enTp211MuPm5iZPT0+7BQAAAPe+fBtuy5YtKz8/P61evdpWlpiYqM2bNys0NFSSFBoaqrNnz2rr1q22OmvWrFFaWprq1at319sMAACAvJWnsyWcP39e+/fvtz0+ePCgtm3bJh8fH913330aOHCgXnvtNVWsWFFly5bVK6+8In9/f7Vt21aSFBQUpObNm6t3796aPn26kpOTFRUVpU6dOjFTAgAAwL9Qnobbn376SY8++qjt8eDBgyVJERERiouL09ChQ3XhwgX16dNHZ8+e1SOPPKLly5erQIECtm3mzp2rqKgoNWnSRI6Ojmrfvr3efffdu94XAAAA5D0HY4zJ60bktcTERHl5eSkhISHXxt826zgmV/YD5KZvFo7I6yYAAJAjt5rX8u2YWwAAACC7CLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwAAAMsg3AIAAMAyCLcAAACwjHwdbkeNGiUHBwe7pUqVKrb1ly9fVmRkpIoWLapChQqpffv2OnHiRB62GAAAAHkpX4dbSapWrZqOHz9uW7777jvbukGDBunLL7/UokWLtH79eh07dkxPPPFEHrYWAAAAeck5rxtwM87OzvLz88tQnpCQoJkzZ2revHlq3LixJCk2NlZBQUH64Ycf9OCDD97tpgIAACCP5fs7t/v27ZO/v7/KlSunLl266MiRI5KkrVu3Kjk5WWFhYba6VapU0X333adNmzbdcJ9JSUlKTEy0WwAAAHDvy9fhtl69eoqLi9Py5cs1bdo0HTx4UPXr19e5c+cUHx8vV1dXeXt7221TokQJxcfH33C/MTEx8vLysi0BAQF3sBcAAAC4W/L1sIQWLVrYfg4JCVG9evUUGBioTz75RO7u7jneb3R0tAYPHmx7nJiYSMAFAACwgHx95/Z63t7eqlSpkvbv3y8/Pz9duXJFZ8+etatz4sSJTMfoXsvNzU2enp52CwAAAO5991S4PX/+vA4cOKCSJUuqdu3acnFx0erVq23r9+zZoyNHjig0NDQPWwkAAIC8kq+HJbzwwgtq06aNAgMDdezYMY0cOVJOTk7q3LmzvLy81LNnTw0ePFg+Pj7y9PRU//79FRoaykwJAAAA/1L5Otz++eef6ty5s06dOqXixYvrkUce0Q8//KDixYtLkiZMmCBHR0e1b99eSUlJCg8P19SpU/O41QAAAMgrDsYYk9eNyGuJiYny8vJSQkJCro2/bdZxTK7sB8hN3ywckddNAAAgR241r91TY24BAACAGyHcAgAAwDIItwAAALAMwi0AAAAsg3ALAAAAyyDcAgAAwDIItwAAALAMwi0AAAAsg3ALAAAAyyDcAgAAwDIItwAAALAMwi0AAAAsg3ALAAAAyyDcAgAAwDIItwAAALAMwi0A3APeeOMNOTg4aODAgTesd/bsWUVGRqpkyZJyc3NTpUqV9PXXX9vWz507VwEBASpSpIgGDx5st+2hQ4dUqVIlJSYm3okuAMBd4ZzXDQAA3NiWLVv0/vvvKyQk5Ib1rly5oqZNm8rX11eLFy9WqVKldPjwYXl7e0uS/vnnH/Xq1UtxcXEqV66cWrVqpcaNG6t169aSpGeffVZvvPGGPD0973SXAOCOIdwCQD52/vx5denSRR988IFee+21G9adNWuWTp8+rY0bN8rFxUWSVKZMGdv6P/74Q15eXurYsaMk6dFHH9WuXbvUunVrzZ8/Xy4uLnriiSfuWF8A4G5gWAIA5GORkZFq1aqVwsLCblr3iy++UGhoqCIjI1WiRAlVr15dY8eOVWpqqiSpYsWKunjxon755RedPn1aW7ZsUUhIiM6cOaNXXnlF77333p3uDgDccdy5BYB8asGCBfr555+1ZcuWW6r/xx9/aM2aNerSpYu+/vpr7d+/X88++6ySk5M1cuRIFSlSRLNnz1a3bt106dIldevWTeHh4erZs6eioqJ08OBBPfbYY0pOTtaoUaPUoUOHO9xDAMh9hFsAyIeOHj2qAQMGaOXKlSpQoMAtbZOWliZfX1/NmDFDTk5Oql27tv766y+NGzdOI0eOlCS1a9dO7dq1s22zfv16bd++XZMnT1aFChU0f/58+fn5qW7dumrQoIF8fX3vSP8A4E4h3AJAPrR161adPHlS999/v60sNTVVGzZs0HvvvaekpCQ5OTnZbVOyZEm5uLjYlQcFBSk+Pl5XrlyRq6urXf2kpCQ9++yzmjNnjvbv36+UlBQ1bNhQklSpUiVt3rxZbdq0uYO9BIDcx5hbAMiHmjRpoh07dmjbtm225YEHHlCXLl20bdu2DMFWkh5++GHt379faWlptrK9e/eqZMmSGYKtJL322mtq3ry57r//fqWmpiolJcW2Ljk52TZWFwDuJdy5BYB8qHDhwqpevbpdmYeHh4oWLWor79atm0qVKqWYmBhJUr9+/fTee+9pwIAB6t+/v/bt26exY8fqueeey7D/33//XQsXLtQvv/wiSapSpYocHR01c+ZM+fn5affu3apTp84d7iUA5D7CLQDco44cOSJHx/97Ay4gIEArVqzQoEGDFBISolKlSmnAgAEaNmyY3XbGGPXp00fjx4+Xh4eHJMnd3V1xcXGKjIxUUlKS3nvvPZUqVequ9gcAcoODMcbkdSPyWmJiory8vJSQkJBrk5c36zgmV/YD5KZvFo7I6yYAAJAjt5rXGHMLAAAAyyDcAgAAwDIYcwsgX6n52qi8bgJgZ9vwUXndBADZwJ1bAAAAWAbhFgAAAJZBuAUAAIBlEG4BAIDlTJkyRWXKlFGBAgVUr149/fjjj1nW/fTTT/XAAw/I29tbHh4eqlmzpubMmWNX5+2335avr698fX31zjvv2K3bvHmzateubfctf8g7fKAMAABYysKFCzV48GBNnz5d9erV08SJExUeHq49e/bI19c3Q30fHx+9/PLLqlKlilxdXbV06VL16NFDvr6+Cg8P1/bt2zVixAgtXbpUxhi1bt1azZo1U3BwsFJSUtS3b1/NmDFDzs7EqvyAO7cAAMBSxo8fr969e6tHjx6qWrWqpk+froIFC2rWrFmZ1m/UqJHatWunoKAglS9fXgMGDFBISIi+++47SdLu3bsVEhKixo0bq0mTJgoJCdHu3bslSePGjVODBg34uup8hHALAAAs48qVK9q6davCwsJsZY6OjgoLC9OmTZtuur0xRqtXr9aePXvUoEEDSVJwcLD27t2rI0eO6PDhw9q7d6+qV6+uAwcOKDY2Vq+99tod6w+yj3ALAAAs459//lFqaqpKlChhV16iRAnFx8dnuV1CQoIKFSokV1dXtWrVSpMnT1bTpk0lSUFBQRo7dqyaNm2qZs2aKSYmRkFBQXrmmWf01ltvacWKFapevbpq1aqlDRs23NH+4eYYHAIAAP71ChcurG3btun8+fNavXq1Bg8erHLlyqlRo0aSpL59+6pv3762+rNnz1bhwoUVGhqqypUra8uWLfrzzz/VqVMnHTx4UG5ubnnUExBuAQCAZRQrVkxOTk46ceKEXfmJEyfk5+eX5XaOjo6qUKGCJKlmzZratWuXYmJibOH2Wv/8849Gjx6tDRs2aPPmzapUqZIqVqyoihUrKjk5WXv37lVwcHCu9gu3jmEJAADAMlxdXVW7dm2tXr3aVpaWlqbVq1crNDT0lveTlpampKSkTNcNGjRIgwYNUunSpZWamqrk5GTbupSUFKWmpua8A7ht3LkFAACWMnjwYEVEROiBBx5Q3bp1NXHiRF24cEE9evSQJHXr1k2lSpVSTEyMJCkmJkYPPPCAypcvr6SkJH399deaM2eOpk2blmHfK1eu1N69ezV79mxJUp06dbR7924tW7ZMR48elZOTkypXrnz3OosMCLcAAMBSOnbsqL///lsjRoxQfHy8atasqeXLl9s+ZHbkyBE5Ov7fm9cXLlzQs88+qz///FPu7u6qUqWKPv74Y3Xs2NFuv5cuXVJUVJQWLlxo27506dKaPHmyevToITc3N82ePVvu7u53r7PIwMEYY/K6EblhypQpGjdunOLj41WjRg1NnjxZdevWvaVtExMT5eXlpYSEBHl6euZKe5p1HJMr+wFy0zcLR+R1E26q5muj8roJgJ1tw0fldRMA6NbzmiXG3KZ/E8nIkSP1888/q0aNGgoPD9fJkyfzumkAAAC4iywRbrP7TSQAAACwpnt+zG36N5FER0fbym72TSRJSUl2n4BMSEiQdPV2d25JSb6ca/sCcktuPsfvlNTLmX86Gcgr98J1I0lD1g3N6yYAdsY1eitX95d+Ld5sRO09H25v9E0k6d/7fL2YmBiNHj06Q3lAQMAdaSOQX3h9FpPXTQDuOV6vv5HXTQDuSTP0/h3Z77lz5+Tl5ZXl+ns+3OZEdHS0Bg8ebHuclpam06dPq2jRonJwcMjDluF6iYmJCggI0NGjR3Ptw37AvwHXDpB9XDf5mzFG586dk7+//w3r3fPhNiffROLm5pbha/G8vb3vVBORCzw9PXmhAXKAawfIPq6b/OtGd2zT3fMfKMutbyIBAADAve+ev3Mr3fybSAAAAPDvYIlwe7NvIsG9y83NTSNHjswwjATAjXHtANnHdWMNlvmGMgAAAOCeH3MLAAAApCPcAgAAwDIItwAAALAMwi0AAAAsg3CLTHXv3l0ODg5ycHCQi4uLypYtq6FDh+ry5cu2Ounrr18WLFggSVq3bp0cHBx09uzZTI8xatSoTLevUqWKJCk4OFh9+/bNdNs5c+bIzc1N//zzj60sPDxcTk5O2rJlS5b9eeMN+6/RXLJkie1b6a7tc2ZLmTJlbvn8If+49vfq6uqqChUqaMyYMUpJSbHVuZXnTmbXQlxc3A2fMw4ODjp06JBGjRqlmjVr2vZpjFFYWJjCw8MzHG/q1Kny9vbWn3/+ecN+pV9f6Uvx4sXVsmVL7dixI8v2X7s0b97cVqdMmTKaOHFipsc5dOhQln374Ycf9M4776hIkSJ2rw3pLl68KE9PT7377ru2svnz58vJyUmRkZFZ9qlatWpKTU21W+ft7a24uLgM/c5sWbdu3Q3PHXLHrbyuSlJqaqomTJig4OBgFShQQEWKFFGLFi30/fff222Xfj1d+9yUpLNnz2br93rtc8HT01N16tTR559/numxrl8KFChg17+2bdtmeZwyZcpkuo833nhDW7dutV0jmWnSpImeeOIJ2+M///xTrq6uql69epZ9KlCggA4fPmxX3rZtW3Xv3j1DvzNbRo0adYOzZi2EW2SpefPmOn78uP744w9NmDBB77//vkaOHGlXJzY2VsePH7dbbvRicL1q1apl2P67776TJPXs2VMLFizQpUuXMmwXGxurxx57TMWKFZMkHTlyRBs3blRUVJRmzZqV6bEKFCigN998U2fOnMl0/aRJk+zacX3/Mgs+uDekP5f37dun559/XqNGjdK4ceMk3dpzJ6troWPHjnbPmdDQUPXu3duuLCAgIMP+HBwcFBsbq82bN+v99//vu9cPHjyooUOHavLkySpduvQt9W3Pnj06fvy4VqxYoaSkJLVq1UpXrlzJtP3XLvPnz7/V0ydJWrVqVYZ91K5dW127dtWFCxf06aefZthm8eLFunLlip566ilb2cyZMzV06FDNnz8/00AsSX/88Yc++uijTNc99NBDdm148sknM/TvoYceylbfkHM3e101xqhTp04aM2aMBgwYoF27dmndunUKCAhQo0aNtGTJErv6zs7OWrVqldauXXtb7Up/7f7pp5/08MMPq0OHDhn+8fP09MzwnL4+PN7MmDFjMuyjf//+ql27tmrUqJHpa8qhQ4e0du1a9ezZ01YWFxenJ598UomJidq8eXOmx3JwcNCIESOybMu1bZg4cWKG/r3wwgvZ6tu9jHCLLLm5ucnPz08BAQFq27atwsLCtHLlSrs63t7e8vPzs1uu/c/3ZpydnTNsnx5Yn3rqKV26dEn/+9//7LY5ePCg1q1bZ/fCEBsbq9atW6tfv36aP39+poE4LCxMfn5+iomJybQtXl5edu24vn/Fixe/5X4hf0l/LgcGBqpfv34KCwvTF198IenWnjtZXQvu7u52zxlXV1cVLFjQrszJySnTNgUEBGjSpEl64YUXdPDgQRlj1LNnTzVr1kxdu3a95b75+vrKz89P999/vwYOHKijR49q9+7dmbb/2qVIkSLZOINS0aJFM+zDxcVFvr6+atOmTaZ/xGfNmqW2bdvKx8dH0tVrd+PGjXrxxRdVqVKlTAOxJPXv318jR45UUlJShnWurq52bXB3d8/QP1dX12z1DTl3s9fVTz75RIsXL9ZHH32kXr16qWzZsqpRo4ZmzJihxx57TL169dKFCxds9T08PPT000/rxRdfvK12pb92V6pUSa+++qpSUlIyBGYHB4cMz+nszo9fuHDhDPvw8PCQdPUGzcKFC3Xx4kW7beLi4lSyZEnbHWpjjGJjY9W1a1f997//1cyZMzM9VlRUlD7++GPt3Lkz0/XXtsHLyytD/woVKpStvt3LCLe4JTt37tTGjRvv6h+NYsWK6fHHH8/wRzMuLk6lS5dWs2bNJP3fC8NTTz2lKlWqqEKFClq8eHGG/Tk5OWns2LGaPHnyTd/yhbW5u7vrypUrt/zcuVZuXgsRERFq0qSJnn76ab333nvauXOn3Z3c7EhISLANCbrb4a5nz55as2aN3V2vP/74Qxs2bMjwT2irVq3k5eWlp556Kss/4gMHDlRKSoomT558x9uO23Oz19V58+apUqVKatOmTYZ1zz//vE6dOpXhpsmoUaO0Y8eOm16LtyIlJcX2PLvb10WXLl2UlJRk1w9jjGbPnq3u3bvb/vFdu3atLl68qLCwMD311FNasGCBXeBP9/DDD6t169a3Hfz/DQi3yNLSpUtVqFAhFShQQMHBwTp58qSGDBliV6dz584qVKiQ3XLkyJFbPsaOHTsybH/tONuePXtq3bp1OnjwoKT/e2GIiIiQo+PVp++qVat08eJF2/jFG/3RbNeunWrWrJlheAX+HYwxWrVqlVasWKHGjRvf8nPnVq6FnJoxY4Z27typgQMHasaMGdl+h6B06dIqVKiQvL29NW/ePD322GO2cevXt//aZezYsdk6zkMPPZRhH+nCw8Pl7++v2NhYW1lcXJwCAgLUpEkTSVJaWpri4uJsQxQ6deqk7777znZtX6tgwYIaOXKkYmJilJCQkK124u670evq3r17FRQUlOl26eV79+61K/f399eAAQP08ssv242Nz470v01ubm4aNGiQypQpoyeffNKuTkJCQobndIsWLbJ1nGHDhmXYx7fffitJ8vHxUbt27exu0Kxdu1aHDh1Sjx49bGUzZ85Up06d5OTkpOrVq6tcuXJatGhRpseLiYnR8uXLbcdA5gi3yNKjjz6qbdu2afPmzYqIiFCPHj3Uvn17uzoTJkzQtm3b7BZ/f/9bPkblypUzbD9mzBjb+qZNm6p06dK2P5qrV6/WkSNH7F4YZs2apY4dO8rZ+eq3SXfu3Fnff/+9Dhw4kOkx33zzTc2ePVu7du265Xbi3nZtOG3RooU6duyoUaNG3fJz51auhZzy9fXVM888o6CgoGyNV0/37bffauvWrYqLi1OlSpU0ffr0DHXS23/tktWHNbOycOHCDPtI5+TkpIiICMXFxckYo7S0NM2ePVs9evSw/RO6cuVKXbhwQS1btpR09Z2Zpk2bZjnOuWfPnipatKjefPPNbLUTeeNGr6s5+SLUYcOG6e+//87y+XEz6X+bli1bpqpVq+rDDz+0DY9JV7hw4QzP6Q8//DBbxxkyZEiGfTzwwAO29U8//bQ2bNhge02ZNWuWGjZsqAoVKki6+mG5Tz/91G5c+o1u0FStWlXdunXj7u1NOOd1A5B/eXh42C7AWbNmqUaNGpo5c6bd24x+fn62OjmR/un1rDg6Oqp79+6aPXu2Ro0apdjYWD366KMqV66cJOn06dP67LPPlJycrGnTptm2S01N1axZs/T6669n2GeDBg0UHh6u6Oho26dMYW2PPvqopk2bJldXV/n7+8vZ2Tlbz51buRZuh7Ozsy1gZ1fZsmXl7e2typUr6+TJk+rYsaM2bNhgV+fa9udUQEDADffx9NNPKyYmRmvWrFFaWpqOHj2a4e7U6dOn5e7ubitLS0vT9u3bNXr0aFsITufs7KzXX39d3bt3V1RU1G21HXdeVq+rlSpVyvJGQnp5pUqVMqzz9vZWdHS0Ro8erdatW2e7Pel/mypUqKDY2Fi1bNlSv//+u3x9fW11HB0db/u6KFas2A330aRJE913332Ki4vTkCFD9Omnn9oNPZo3b54uX76sevXq2crS/0Hcu3dvpudm9OjRqlSpUoYP4+H/cOcWt8TR0VEvvfSShg8fnukHbu6kHj166OjRo/r000/12Wef2QWKuXPnqnTp0vr111/t/nN+5513FBcXl2E6oXRvvPGGvvzyS23atOludQN5KD3c3XfffbYQmdPnTl5eCzcTGRmpnTt36rPPPrvrxy5fvrwaNmyoWbNmKTY2VmFhYQoMDJQknTp1Sp9//rkWLFhgd65/+eUXnTlzRt98802m+/zPf/6jatWqafTo0XezK8ihzF5XO3XqpH379unLL7/MUP+dd95R0aJF1bRp00z3179/fzk6OmrSpEm31a66deuqdu3amd7suNMcHR3Vo0cPzZ49W/PmzZOrq6s6dOhgWz9z5kw9//zzdtfFr7/+qvr162d51zogIEBRUVF66aWXsnyd+rfjzi1u2X/+8x8NGTJEU6ZMsU0pcvbsWcXHx9vVK1y4sO3TotLVcbWFCxe2PXZwcFCNGjUkXR3sf/32Dg4Odp9YLVu2rBo3bqw+ffrIzc3Nbm7AmTNnqkOHDhnmBgwICFB0dLSWL1+uVq1aZehLcHCwunTpYjf/Jv5dcvrckTK/Fm7m0qVLdm/lS1evlfLly+eo/ZkpWLCgevfurZEjR6pt27a2uUaTkpIyXGfOzs62mUkk6a+//srQvvRwKl0NqNfvw9vb2252lJ49e6p3796Sro65TTdnzhwVLVpUTz75pN38p5LUsmVLzZw5M8PcpuneeOONTOcDRv6T2etqp06dtGjRIkVERGjcuHFq0qSJEhMTNWXKFH3xxRdatGiR3d+LaxUoUECjR4/OdE7k7Bo4cKDatWunoUOHqlSpUpKu3iG9/jktXR0qlP5OQkJCQobromjRorYp/s6dO5dhHwULFpSnp6ftcY8ePTRmzBi99NJL6ty5s+3di23btunnn3/W3LlzM4yT79y5s8aMGaPXXnst03d1oqOj9cEHH+jgwYPq2LFjNs/Gv4ABMhEREWEef/zxDOUxMTGmePHi5vz580ZSpktMTIwxxpi1a9dmut7JyckYY8zIkSMzXe/m5pbhuPPmzTOSzLPPPmsr++mnn4wk8+OPP2bahxYtWph27dpl2Z+DBw8aV1dXk9VlIMl89tlnNztVyOcy+93f7nPHGPtrIV3Dhg3NgAEDMtTN6rnepEkT2/oaNWpkq1/p19eZM2fsyo8cOWKcnZ3NwoULbe3P7NiVK1e2bRMYGJhpnTlz5piDBw9mea3Pnz/f7tgXL140Xl5exsfHx1y+fNlWHhwcbHftXmvhwoXG1dXV/P3331n2qVmzZkaSiY2NzbB9Vr8f3Hm3+rqanJxsxo0bZ6pVq2ZcXV2Np6enCQ8PN999953dtrGxscbLy8uuLCUlxVStWtVIMmvXrr2ldmX22p2WlmaqVKli+vXrZztWVs/r48eP2/qX2fqePXsaY7K+bp555pkMbUp/Dl/7mhMVFWWqVq2aaR+OHz9uHB0dzeeff55ln8aOHWskmYiIiAzbZ3Yu/00cjMnBSG8AAAAgH2LMLQAAACyDcAsA+VCLFi0yzJ+Z0zlqAasYO3ZsltdFdueohXUxLAEA8qG//vory9kYfHx8MszZCfwbnD59WqdPn850nbu7u+3DYvh3I9wCAADAMhiWAAAAAMsg3AIAAMAyCLcAAACwDMItAAAALINwCwD5yLp16+Tg4KCzZ8/mdVMA4J5EuAWAXNS9e3c5ODjIwcFBLi4uKlu2rIYOHarLly/nddMA4F/BOa8bAABW07x5c8XGxio5OVlbt25VRESEHBwc9Oabb+Z10wDA8rhzCwC5zM3NTX5+fgoICFDbtm0VFhamlStXSpLS0tIUExOjsmXLyt3dXTVq1NDixYtvuL/vvvtO9evXl7u7uwICAvTcc8/pwoULkqSXXnpJ9erVy7BNjRo1NGbMGEnSli1b1LRpUxUrVkxeXl5q2LChfv75Z7v6Dg4O+vDDD9WuXTsVLFhQFStW1BdffGFX57ffflPr1q3l6empwoULq379+jpw4IBt/YcffqigoCAVKFBAVapU0dSpU7N/8gDgNhFuAeAO2rlzpzZu3ChXV1dJUkxMjD766CNNnz5dv/32mwYNGqSnnnpK69evz3T7AwcOqHnz5mrfvr22b9+uhQsX6rvvvlNUVJQkqUuXLvrxxx/tQuZvv/2m7du367///a8k6dy5c4qIiNB3332nH374QRUrVlTLli117tw5u2ONHj1aTz75pLZv366WLVuqS5cutm+D+uuvv9SgQQO5ublpzZo12rp1q55++mmlpKRIkubOnasRI0bo9ddf165duzR27Fi98sormj17du6eUAC4GQMAyDURERHGycnJeHh4GDc3NyPJODo6msWLF5vLly+bggULmo0bN9pt07NnT9O5c2djjDFr1641ksyZM2ds6/r06WNX/9tvvzWOjo7m0qVLxhhjatSoYcaMGWNbHx0dberVq5dlG1NTU03hwoXNl19+aSuTZIYPH257fP78eSPJLFu2zLbPsmXLmitXrmS6z/Lly5t58+bZlb366qsmNDQ0y3YAwJ3AmFsAyGWPPvqopk2bpgsXLmjChAlydnZW+/bt9dtvv+nixYtq2rSpXf0rV66oVq1ame7r119/1fbt2zV37lxbmTFGaWlpOnjwoIKCgtSlSxfNmjVLr7zyiowxmj9/vgYPHmyrf+LECQ0fPlzr1q3TyZMnlZqaqosXL+rIkSN2xwoJCbH97OHhIU9PT508eVKStG3bNtWvX18uLi4Z2njhwgUdOHBAPXv2VO/evW3lKSkp8vLyysaZA4DbR7gFgFzm4eGhChUqSJJmzZqlGjVqaObMmapevbok6auvvlKpUqXstnFzc8t0X+fPn9czzzyj5557LsO6++67T5LUuXNnDRs2TD///LMuXbqko0ePqmPHjrZ6EREROnXqlCZNmqTAwEC5ubkpNDRUV65csdvf9cHVwcFBaWlpkiR3d/cs+3v+/HlJ0gcffJBh/K+Tk1OW2wHAnUC4BYA7yNHRUS+99JIGDx6svXv3ys3NTUeOHFHDhg1vafv7779fv//+uy0sZ6Z06dJq2LCh5s6dq0uXLqlp06by9fW1rf/+++81depUtWzZUpJ09OhR/fPPP9nqR0hIiGbPnq3k5OQMIbhEiRLy9/fXH3/8oS5dumRrvwCQ2/hAGQDcYf/5z3/k5OSk999/Xy+88IIGDRqk2bNn68CBA/r55581efLkLD94NWzYMG3cuFFRUVHatm2b9u3bp88//9z2gbJ0Xbp00YIFC7Ro0aIMAbNixYqaM2eOdu3apc2bN6tLly43vBObmaioKCUmJqpTp0766aeftG/fPs2ZM0d79uyRdPXDaDExMXr33Xe1d+9e7dixQ7GxsRo/fny2jgMAt4twCwB3mLOzs6KiovTWW28pOjpar7zyimJiYhQUFKTmzZvrq6++UtmyZTPdNiQkROvXr9fevXtVv3591apVSyNGjJC/v79dvQ4dOujUqVO6ePGi2rZta7du5syZOnPmjO6//3517dpVzz33nN2d3VtRtGhRrVmzRufPn1fDhg1Vu3ZtffDBB7a7uL169dKHH36o2NhYBQcHq2HDhoqLi8uyXwBwpzgYY0xeNwIAAADIDdy5BQAAgGUQbgEAAGAZhFsAAABYBuEWAAAAlkG4BQAAgGUQbgEAAGAZhFsAAABYBuEWAAAAlkG4BQAAgGUQbgEAAGAZhFsAAABYxv8DmA6Ghk2ZHiQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# Calculate counts and percentages\n",
    "counts = df_eval['relevance'].value_counts()\n",
    "print(counts)\n",
    "percentages = counts / counts.sum() * 100\n",
    "\n",
    "# Create bar plot\n",
    "plt.figure(figsize=(8, 5))\n",
    "ax = sns.countplot(data=df_eval, x='relevance', palette='viridis')\n",
    "\n",
    "# Add percentage labels on top of the bars\n",
    "for p in ax.patches:\n",
    "    height = p.get_height()\n",
    "    ax.annotate(f'{height / counts.sum() * 100:.1f}%', \n",
    "                (p.get_x() + p.get_width() / 2., height), \n",
    "                ha='center', va='bottom')\n",
    "\n",
    "plt.title('Distribution of Relevance For mixtral-8x7b-32768')\n",
    "plt.xlabel('Relevance')\n",
    "plt.ylabel('Count')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>relevance</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mental illnesses can significantly impact a pe...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>How do mental illnesses affect a person's dail...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer is highly relevant to the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sure, I'd be happy to help answer your questio...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>What are some examples of serious mental illne...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer provides a detailed list ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mental illness is not a reflection of a person...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>Why is it a misconception to associate mental ...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mental illnesses are treated using a variety o...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>How are mental illnesses treated, and what is ...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer fully addresses the quest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yes, a person with a mental illness can become...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>Can a person with mental illness become indepe...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer       id  \\\n",
       "0  Mental illnesses can significantly impact a pe...  1590140   \n",
       "1  Sure, I'd be happy to help answer your questio...  1590140   \n",
       "2  Mental illness is not a reflection of a person...  1590140   \n",
       "3  Mental illnesses are treated using a variety o...  1590140   \n",
       "4  Yes, a person with a mental illness can become...  1590140   \n",
       "\n",
       "                                            question relevance  \\\n",
       "0  How do mental illnesses affect a person's dail...  RELEVANT   \n",
       "1  What are some examples of serious mental illne...  RELEVANT   \n",
       "2  Why is it a misconception to associate mental ...  RELEVANT   \n",
       "3  How are mental illnesses treated, and what is ...  RELEVANT   \n",
       "4  Can a person with mental illness become indepe...  RELEVANT   \n",
       "\n",
       "                                         explanation  \n",
       "0  The generated answer is highly relevant to the...  \n",
       "1  The generated answer provides a detailed list ...  \n",
       "2  The generated answer directly addresses the qu...  \n",
       "3  The generated answer fully addresses the quest...  \n",
       "4  The generated answer directly addresses the qu...  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter relevant entries\n",
    "relevant_answers = df_eval[df_eval['relevance'] == 'RELEVANT']\n",
    "relevant_answers.head(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>relevance</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>The context provided does not include informat...</td>\n",
       "      <td>3284724</td>\n",
       "      <td>What are the different income levels for provi...</td>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not provide any info...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                answer       id  \\\n",
       "208  The context provided does not include informat...  3284724   \n",
       "\n",
       "                                              question     relevance  \\\n",
       "208  What are the different income levels for provi...  NON_RELEVANT   \n",
       "\n",
       "                                           explanation  \n",
       "208  The generated answer does not provide any info...  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter non-relevant entries\n",
    "non_relevant_answers = df_eval[df_eval['relevance'] == 'NON_RELEVANT']\n",
    "non_relevant_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_relevant_answers[['question','answer','explanation']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Set display options to show full content\n",
    "pd.set_option('display.max_colwidth', None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysing Partly Relevant Answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "partly_relevant_answers = df_eval[df_eval['relevance'] == 'PARTLY_RELEVANT']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "partly_relevant_answers[['question','answer','explanation']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_eval.to_csv('../dataset/rag-eval-mistral.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating Mistralai Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e3ded5a907f4fe58c4fbe4e590c9459",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/366 [00:01<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations_llama = []\n",
    "\n",
    "for record in tqdm(ground_truth):\n",
    "    question = record['question']\n",
    "    answer_llm = rag(question, model='llama3-70b-8192') \n",
    "\n",
    "    prompt = prompt2_template.format(\n",
    "        question=question,\n",
    "        answer_llm=answer_llm\n",
    "    )\n",
    "\n",
    "    evaluation = llm(prompt,model='llama3-70b-8192')\n",
    "    evaluation = json.loads(evaluation)\n",
    "    \n",
    "    evaluations_llama.append((record, answer_llm, evaluation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_llama = pd.DataFrame(evaluations_llama, columns=['record', 'answer', 'evaluation'])\n",
    "\n",
    "df_eval_llama['id'] = df_eval_llama.record.apply(lambda d: d['id'])\n",
    "df_eval_llama['question'] = df_eval_llama.record.apply(lambda d: d['question'])\n",
    "\n",
    "df_eval_llama['relevance'] = df_eval_llama.evaluation.apply(lambda d: d['Relevance'])\n",
    "df_eval_llama['explanation'] = df_eval_llama.evaluation.apply(lambda d: d['Explanation'])\n",
    "\n",
    "del df_eval_llama['record']\n",
    "del df_eval_llama['evaluation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>relevance</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Based on the provided context, mental illnesse...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>How do mental illnesses affect a person's dail...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Based on the provided context, some examples o...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>What are some examples of serious mental illne...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>It is a misconception to associate mental illn...</td>\n",
       "      <td>1590140</td>\n",
       "      <td>Why is it a misconception to associate mental ...</td>\n",
       "      <td>RELEVANT</td>\n",
       "      <td>The generated answer directly addresses the qu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer       id  \\\n",
       "0  Based on the provided context, mental illnesse...  1590140   \n",
       "1  Based on the provided context, some examples o...  1590140   \n",
       "2  It is a misconception to associate mental illn...  1590140   \n",
       "\n",
       "                                            question relevance  \\\n",
       "0  How do mental illnesses affect a person's dail...  RELEVANT   \n",
       "1  What are some examples of serious mental illne...  RELEVANT   \n",
       "2  Why is it a misconception to associate mental ...  RELEVANT   \n",
       "\n",
       "                                         explanation  \n",
       "0  The generated answer directly addresses the qu...  \n",
       "1  The generated answer directly addresses the qu...  \n",
       "2  The generated answer directly addresses the qu...  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval_llama.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_llama.to_csv('../dataset/rag_eval_llama.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relevance\n",
      "RELEVANT           317\n",
      "NON_RELEVANT        31\n",
      "PARTLY_RELEVANT     18\n",
      "Name: count, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\AppData\\Local\\Temp\\ipykernel_16892\\4118569142.py:8: FutureWarning: \n",
      "\n",
      "Passing `palette` without assigning `hue` is deprecated and will be removed in v0.14.0. Assign the `x` variable to `hue` and set `legend=False` for the same effect.\n",
      "\n",
      "  ax = sns.countplot(data=df_eval, x='relevance', palette='viridis')\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArcAAAHWCAYAAABt3aEVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVv0lEQVR4nO3deVwV9eL/8fcBAREF3ABRRNxRQU3NaHFFcc/Sm3pN0XANLLXUi9fcMqms1Kw0S4FM0/S2est9yzQzu+SSe26piKmAK+v8/ujH+XoCFBAFp9fz8ZjHw/OZz3zm8xlmDm+Hz5ljMQzDEAAAAGACdkXdAQAAAKCwEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BE5k8ebIsFss92VerVq3UqlUr6+tNmzbJYrFoxYoV92T/AwYMULVq1e7JvgrqypUrGjRokLy8vGSxWDRy5Mh7uv+//oxQ9HI6by0WiyZPnlwk/QHMiHALFFMxMTGyWCzWpWTJkvL29lZISIjefvttXb58uVD2c+bMGU2ePFlxcXGF0l5hKs59y4vp06crJiZGw4cP16JFi9SvX79c61arVs3m5+3i4qIHH3xQH3300T3s8d9Dq1atbI71zcuBAweKunsA7lCJou4AgFubOnWq/Pz8lJaWpvj4eG3atEkjR47UW2+9pa+++kqBgYHWuhMmTNC//vWvfLV/5swZTZkyRdWqVVOjRo3yvN2aNWvytZ+CuFXfPvjgA2VmZt71PtyJDRs26KGHHtKkSZPyVL9Ro0Z64YUXJElnz57Vhx9+qNDQUKWkpGjw4MF3s6t/O1WqVFFUVFS2cm9v7yLoDYDCRLgFirmOHTuqadOm1teRkZHasGGDunTpom7dumn//v1ydnaWJJUoUUIlStzdy/ratWsqVaqUHB0d7+p+bsfBwaFI958XCQkJqlevXp7rV65cWU8//bT19YABA1S9enXNnDmTcFvI3NzcbI51YTEMQzdu3LBekwDuPaYlAPehNm3a6KWXXtKJEyf08ccfW8tzmnO7du1aPfroo3J3d1fp0qVVp04djR8/XtKf82SbNWsmSRo4cKD1T7MxMTGS/vzzbYMGDbRr1y61aNFCpUqVsm6b23zOjIwMjR8/Xl5eXnJxcVG3bt106tQpmzrVqlXTgAEDsm17c5u361tOcxevXr2qF154QT4+PnJyclKdOnX0xhtvyDAMm3oWi0URERH64osv1KBBAzk5Oal+/fpatWpVzgf8LxISEhQWFiZPT0+VLFlSDRs2VGxsrHV91vzjY8eO6b///a+178ePH89T+1kqVqyounXr6ujRozblmZmZmjVrlurXr6+SJUvK09NTQ4cO1aVLl27bZkpKiiZNmqSaNWvKyclJPj4+Gjt2rFJSUqx1GjRooNatW2fbNjMzU5UrV1bPnj2tZW+88YYefvhhlS9fXs7OzmrSpEmO867zc8xPnz6tsLAweXt7y8nJSX5+fho+fLhSU1OtdRITEzVy5Ejrz7pmzZp67bXXCu1ufnp6ul5++WXVqFFDTk5OqlatmsaPH29znKQ/z+UuXbpo9erVatq0qZydnfX+++/f0b5PnDihZ599VnXq1JGzs7PKly+vf/zjH9nOn6ypS1u3btVzzz2nihUryt3dXUOHDlVqaqoSExPVv39/lS1bVmXLltXYsWOzXQt5/fkB9xPu3AL3qX79+mn8+PFas2ZNrnf19u3bpy5duigwMFBTp06Vk5OTjhw5ou+//16S5O/vr6lTp2rixIkaMmSIHnvsMUnSww8/bG3jwoUL6tixo3r37q2nn35anp6et+zXK6+8IovFonHjxikhIUGzZs1ScHCw4uLi8nU3Ky99u5lhGOrWrZs2btyosLAwNWrUSKtXr9aYMWN0+vRpzZw506b+1q1b9dlnn+nZZ59VmTJl9Pbbb6tHjx46efKkypcvn2u/rl+/rlatWunIkSOKiIiQn5+fli9frgEDBigxMVHPP/+8/P39tWjRIo0aNUpVqlSxTjWoWLFinscv/Rmwfv/9d5UtW9amfOjQoYqJidHAgQP13HPP6dixY3rnnXf0v//9T99//32ud7UzMzPVrVs3bd26VUOGDJG/v7/27NmjmTNn6tChQ/riiy8kSb169dLkyZMVHx8vLy8vm2N25swZ9e7d21o2e/ZsdevWTX379lVqaqqWLl2qf/zjH1q5cqU6d+6c72N+5swZPfjgg0pMTNSQIUNUt25dnT59WitWrNC1a9fk6Oioa9euqWXLljp9+rSGDh2qqlWratu2bYqMjNTZs2c1a9as2x7bjIwM/fHHHzZlJUuWVOnSpSVJgwYNUmxsrHr27KkXXnhBO3bsUFRUlPbv36/PP//cZruDBw+qT58+Gjp0qAYPHqw6dercdv+3snPnTm3btk29e/dWlSpVdPz4cc2dO1etWrXSr7/+qlKlStnUHzFihLy8vDRlyhT98MMPmj9/vtzd3bVt2zZVrVpV06dP1zfffKMZM2aoQYMG6t+/v3Xb/Pz8gPuGAaBYio6ONiQZO3fuzLWOm5ub0bhxY+vrSZMmGTdf1jNnzjQkGefPn8+1jZ07dxqSjOjo6GzrWrZsaUgy5s2bl+O6li1bWl9v3LjRkGRUrlzZSE5OtpZ/+umnhiRj9uzZ1jJfX18jNDT0tm3eqm+hoaGGr6+v9fUXX3xhSDKmTZtmU69nz56GxWIxjhw5Yi2TZDg6OtqU/fLLL4YkY86cOdn2dbNZs2YZkoyPP/7YWpaammoEBQUZpUuXthm7r6+v0blz51u2d3Pd9u3bG+fPnzfOnz9v7Nmzx+jXr58hyQgPD7fW++677wxJxuLFi222X7VqVbbyvx7PRYsWGXZ2dsZ3331ns+28efMMScb3339vGIZhHDx4MMdj8eyzzxqlS5c2rl27Zi27+d9Zx6JBgwZGmzZtbMrzesz79+9v2NnZ5XjeZ2ZmGoZhGC+//LLh4uJiHDp0yGb9v/71L8Pe3t44efJktm1vlnVe/3XJOifj4uIMScagQYNstnvxxRcNScaGDRusZb6+voYkY9WqVbfcZ5a/nreG8eexmTRpkvX1X4+pYRjG9u3bDUnGRx99ZC3Leo8ICQmxHhvDMIygoCDDYrEYw4YNs5alp6cbVapUsTkfctpXbj8/4H7CtATgPla6dOlbPjXB3d1dkvTll18W+M+1Tk5OGjhwYJ7r9+/fX2XKlLG+7tmzpypVqqRvvvmmQPvPq2+++Ub29vZ67rnnbMpfeOEFGYahb7/91qY8ODhYNWrUsL4ODAyUq6urfvvtt9vux8vLS3369LGWOTg46LnnntOVK1e0efPmAo9hzZo1qlixoipWrKiAgAAtWrRIAwcO1IwZM6x1li9fLjc3N7Vr105//PGHdWnSpIlKly6tjRs35tr+8uXL5e/vr7p169ps26ZNG0myblu7dm01atRIy5Yts26bkZGhFStWqGvXrjZ34G/+96VLl5SUlKTHHntMP//8c7b93+6YZ2Zm6osvvlDXrl1t5plnyZpys3z5cj322GMqW7aszTiCg4OVkZGhLVu23PpA68/pBGvXrrVZxo4dK0nWc3X06NE222Tdgf/vf/9rU+7n56eQkJDb7jOvbj6maWlpunDhgmrWrCl3d/ccj2tYWJjNdKTmzZvLMAyFhYVZy+zt7dW0adNs53d+fn7A/YJpCcB97MqVK/Lw8Mh1fa9evfThhx9q0KBB+te//qW2bdvqySefVM+ePWVnl7f/21auXDlfHx6rVauWzWuLxaKaNWvme75pfp04cULe3t42wVr6c3pD1vqbVa1aNVsbZcuWve281RMnTqhWrVrZjl9u+8mP5s2ba9q0acrIyNDevXs1bdo0Xbp0yeb4Hz58WElJSbn+3BMSEnJt//Dhw9q/f3+u0yNu3rZXr14aP368Tp8+rcqVK2vTpk1KSEhQr169bLZZuXKlpk2bpri4OJv5qDk9b/l2x/z8+fNKTk5WgwYNch1D1jh2796dp3HkxsXFRcHBwTmuO3HihOzs7FSzZk2bci8vL7m7u2f7Gfv5+d12f/lx/fp1RUVFKTo6WqdPn7aZJ5uUlJSt/l+Pq5ubmyTJx8cnW/lfz+/8/PyA+wXhFrhP/f7770pKSsr2C/hmzs7O2rJlizZu3Kj//ve/WrVqlZYtW6Y2bdpozZo1sre3v+1+7sanvnP7xZmRkZGnPhWG3PZj/OUDN/dShQoVrIErJCREdevWVZcuXTR79mzrXcTMzEx5eHho8eLFObZxq3m9mZmZCggI0FtvvZXj+pvDUK9evRQZGanly5dr5MiR+vTTT+Xm5qYOHTpY63z33Xfq1q2bWrRooffee0+VKlWSg4ODoqOjtWTJkmztF9Yxz8zMVLt27ax3Wv+qdu3a+WovN3kNeIV9jYwYMULR0dEaOXKkgoKC5ObmJovFot69e+f4F5jcjmtO5Tcf6/z+/ID7BeEWuE8tWrRIkm7751A7Ozu1bdtWbdu21VtvvaXp06fr3//+tzZu3Kjg4OBCv0Nz+PBhm9eGYejIkSM2z+MtW7asEhMTs2174sQJVa9e3fo6P33z9fXVunXrdPnyZZu7t1kP5ff19c1zW7fbz+7du5WZmWlz97aw9yNJnTt3VsuWLTV9+nQNHTpULi4uqlGjhtatW6dHHnkk36GqRo0a+uWXX9S2bdvbHls/Pz89+OCDWrZsmSIiIvTZZ5+pe/fucnJystb5z3/+o5IlS2r16tU25dHR0fkb6P9XsWJFubq6au/evbcdx5UrV3K983qnfH19lZmZqcOHD1vvyEvSuXPnlJiYWKg/45ysWLFCoaGhevPNN61lN27cyPGauROF/fMDigvm3AL3oQ0bNujll1+Wn5+f+vbtm2u9ixcvZivL+jKErD9Buri4SFKh/eL86KOPbOYBr1ixQmfPnlXHjh2tZTVq1NAPP/xg82inlStXZntkWH761qlTJ2VkZOidd96xKZ85c6YsFovN/u9Ep06dFB8fbzMfNT09XXPmzFHp0qXVsmXLQtlPlnHjxunChQv64IMPJElPPfWUMjIy9PLLL2erm56efstj9dRTT+n06dPWtm52/fp1Xb161aasV69e+uGHH7Rw4UL98ccf2aYk2Nvby2KxKCMjw1p2/Phx61MX8svOzk7du3fX119/rZ9++inb+qy7jk899ZS2b9+u1atXZ6uTmJio9PT0Au0/S6dOnSQp21MXsu543+2nCNjb22e7mz1nzhyb41xY+ynMnx9QXHDnFijmvv32Wx04cEDp6ek6d+6cNmzYoLVr18rX11dfffWVSpYsmeu2U6dO1ZYtW9S5c2f5+voqISFB7733nqpUqaJHH31U0p9B093dXfPmzVOZMmXk4uKi5s2bF3geYbly5fToo49q4MCBOnfunGbNmqWaNWvaPK5s0KBBWrFihTp06KCnnnpKR48e1ccff2zzYaP89q1r165q3bq1/v3vf+v48eNq2LCh1qxZoy+//FIjR47M1nZBDRkyRO+//74GDBigXbt2qVq1alqxYoW+//57zZo1K9uc3zvVsWNHNWjQQG+99ZbCw8PVsmVLDR06VFFRUYqLi1P79u3l4OCgw4cPa/ny5Zo9e7bNc2hv1q9fP3366acaNmyYNm7cqEceeUQZGRk6cOCAPv30U+uzWrM89dRTevHFF/Xiiy+qXLly2e6Udu7cWW+99ZY6dOigf/7zn0pISNC7776rmjVravfu3QUa7/Tp07VmzRq1bNnS+riys2fPavny5dq6davc3d01ZswYffXVV+rSpYsGDBigJk2a6OrVq9qzZ49WrFih48ePq0KFCgXavyQ1bNhQoaGhmj9/vhITE9WyZUv9+OOPio2NVffu3XN8BnBh6tKlixYtWiQ3NzfVq1dP27dv17p16275iLqCuBs/P6BYKLLnNAC4pazH/GQtjo6OhpeXl9GuXTtj9uzZNo+cyvLXR4GtX7/eePzxxw1vb2/D0dHR8Pb2Nvr06ZPtEUpffvmlUa9ePaNEiRI2j95q2bKlUb9+/Rz7l9ujwD755BMjMjLS8PDwMJydnY3OnTsbJ06cyLb9m2++aVSuXNlwcnIyHnnkEeOnn37K1uat+pbTI5UuX75sjBo1yvD29jYcHByMWrVqGTNmzLB5TJJhGNker5Ult0eU/dW5c+eMgQMHGhUqVDAcHR2NgICAHB9Xlt9HgeVWNyYmJtsj0ebPn280adLEcHZ2NsqUKWMEBAQYY8eONc6cOWOtk9PxTE1NNV577TWjfv36hpOTk1G2bFmjSZMmxpQpU4ykpKRs+37kkUdyfCxWlgULFhi1atUynJycjLp16xrR0dHZzkPDyN8xP3HihNG/f3+jYsWKhpOTk1G9enUjPDzcSElJsda5fPmyERkZadSsWdNwdHQ0KlSoYDz88MPGG2+8YaSmpubY15uPS27ndZa0tDRjypQphp+fn+Hg4GD4+PgYkZGRxo0bN7L1P68/Y8PI26PALl26ZD2/SpcubYSEhBgHDhzIdqxye1xg1vH/6yMAQ0NDDRcXF5uyvP78gPuJxTCK8NMTAAAAQCFizi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0+BLHPTn95SfOXNGZcqUKfSvIgUAAMCdMwxDly9flre3t83Xn/8V4VbSmTNn5OPjU9TdAAAAwG2cOnVKVapUyXU94Vayfl3mqVOn5OrqWsS9AQAAwF8lJyfLx8fntl9zTriVrFMRXF1dCbcAAADF2O2mkPKBMgAAAJgG4RYAAACmQbgFAACAaRBuUeguX76skSNHytfXV87Oznr44Ye1c+dO6/oBAwbIYrHYLB06dLhlm1u2bFHXrl3l7e0ti8WiL774IludN954Qx4eHvLw8NCbb75ps27Hjh1q0qSJ0tPTC2WMAACgeOIDZSh0gwYN0t69e7Vo0SJ5e3vr448/VnBwsH799VdVrlxZktShQwdFR0dbt3Fycrplm1evXlXDhg31zDPP6Mknn8y2fvfu3Zo4caJWrlwpwzDUpUsXtW/fXgEBAUpPT9ewYcM0f/58lSjBKQ8AgJnxmx6F6vr16/rPf/6jL7/8Ui1atJAkTZ48WV9//bXmzp2radOmSfozzHp5eeW53Y4dO6pjx465rj9w4IACAwPVpk0bSVJgYKAOHDiggIAAzZgxQy1atFCzZs3uYGQAAOB+QLhFoUpPT1dGRoZKlixpU+7s7KytW7daX2/atEkeHh4qW7as2rRpo2nTpql8+fIF3m9AQIAOHTqkkydPyjAMHTp0SA0aNNDRo0cVHR2tXbt2FbhtAABw/2DOLQpVmTJlFBQUpJdffllnzpxRRkaGPv74Y23fvl1nz56V9OeUhI8++kjr16/Xa6+9ps2bN6tjx47KyMgo8H79/f01ffp0tWvXTu3bt1dUVJT8/f01dOhQvf7661q9erUaNGigxo0ba8uWLYU1XAAAUMxw5xaFbtGiRXrmmWdUuXJl2dvb64EHHlCfPn2sd0979+5trRsQEKDAwEDVqFFDmzZtUtu2bQu832HDhmnYsGHW17GxsdawXadOHe3cuVO///67evfurWPHjt12ni8AALj/FOmd27lz5yowMND6zWBBQUH69ttvretbtWqV7VP1N4cXSTp58qQ6d+6sUqVKycPDQ2PGjOET8UWsRo0a2rx5s65cuaJTp07pxx9/VFpamqpXr55j/erVq6tChQo6cuRIofXhjz/+0JQpUzRnzhzt2LFDtWvXVq1atdS6dWulpaXp0KFDhbYvAABQfBTpndsqVaro1VdfVa1atWQYhmJjY/X444/rf//7n+rXry9JGjx4sKZOnWrdplSpUtZ/Z2RkqHPnzvLy8tK2bdt09uxZ9e/fXw4ODpo+ffo9Hw9subi4yMXFRZcuXdLq1av1+uuv51jv999/14ULF1SpUqVC2/eoUaM0atQoValSRTt37lRaWpp1Xda8YAAAYD5FGm67du1q8/qVV17R3Llz9cMPP1jDbalSpXL9VP2aNWv066+/at26dfL09FSjRo308ssva9y4cZo8ebIcHR3v+hiQ3erVq2UYhurUqaMjR45ozJgxqlu3rgYOHKgrV65oypQp6tGjh7y8vHT06FGNHTtWNWvWVEhIiLWNtm3b6oknnlBERIQk6cqVKzZ3do8dO6a4uDiVK1dOVatWtdn/2rVrdejQIcXGxkqSmjVrpgMHDujbb7/VqVOnZG9vrzp16tyDIwEAAO61YvOBsoyMDC1dulRXr15VUFCQtXzx4sWqUKGCGjRooMjISF27ds26bvv27QoICJCnp6e1LCQkRMnJydq3b1+u+0pJSVFycrLNgsKTlJSk8PBw1a1bV/3799ejjz6q1atXy8HBQfb29tq9e7e6deum2rVrKywsTE2aNNF3331nMwf26NGj+uOPP6yvf/rpJzVu3FiNGzeWJI0ePVqNGzfWxIkTbfZ9/fp1RURE6P3335ed3Z+nd5UqVTRnzhwNHDhQr7zyimJjY+Xs7HwPjgQAALjXLIZhGEXZgT179igoKEg3btxQ6dKltWTJEnXq1EmSNH/+fPn6+srb21u7d+/WuHHj9OCDD+qzzz6TJA0ZMkQnTpzQ6tWrre1du3ZNLi4u+uabb3J9LurkyZM1ZcqUbOVJSUlydXW9C6MEAADAnUhOTpabm9tt81qRPy2hTp06iouLU1JSklasWKHQ0FBt3rxZ9erV05AhQ6z1AgICVKlSJbVt21ZHjx5VjRo1CrzPyMhIjR492vo6OTlZPj4+dzQOAAAAFL0iD7eOjo6qWbOmJKlJkybauXOnZs+erffffz9b3ebNm0uSjhw5oho1asjLy0s//vijTZ1z585J0i2//crJyemuPwaqfa+pt68E3GNrlk28fSUAAO5jxWbObZbMzEylpKTkuC4uLk6SrJ+qDwoK0p49e5SQkGCts3btWrm6uqpevXp3va8AAAAoXor0zm1kZKQ6duyoqlWr6vLly1qyZIk2bdqk1atX6+jRo9b5t+XLl9fu3bs1atQotWjRQoGBgZKk9u3bq169eurXr59ef/11xcfHa8KECQoPD+cB/QAAAH9DRRpuExIS1L9/f509e1Zubm4KDAzU6tWr1a5dO506dUrr1q3TrFmzdPXqVfn4+KhHjx6aMGGCdXt7e3utXLlSw4cPV1BQkFxcXBQaGmrzXFwAAAD8fRRpuF2wYEGu63x8fLR58+bbtuHr66tvvvmmMLsFAACA+1Sxm3MLAAAAFBThFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZRpOF27ty5CgwMlKurq1xdXRUUFKRvv/3Wuv7GjRsKDw9X+fLlVbp0afXo0UPnzp2zaePkyZPq3LmzSpUqJQ8PD40ZM0bp6en3eigAAAAoBoo03FapUkWvvvqqdu3apZ9++klt2rTR448/rn379kmSRo0apa+//lrLly/X5s2bdebMGT355JPW7TMyMtS5c2elpqZq27Ztio2NVUxMjCZOnFhUQwIAAEARshiGYRR1J25Wrlw5zZgxQz179lTFihW1ZMkS9ezZU5J04MAB+fv7a/v27XrooYf07bffqkuXLjpz5ow8PT0lSfPmzdO4ceN0/vx5OTo65mmfycnJcnNzU1JSklxdXQtlHO17TS2UdoDCtGYZ//EDANyf8prXis2c24yMDC1dulRXr15VUFCQdu3apbS0NAUHB1vr1K1bV1WrVtX27dslSdu3b1dAQIA12EpSSEiIkpOTrXd/c5KSkqLk5GSbBQAAAPe/Ig+3e/bsUenSpeXk5KRhw4bp888/V7169RQfHy9HR0e5u7vb1Pf09FR8fLwkKT4+3ibYZq3PWpebqKgoubm5WRcfH5/CHRQAAACKRJGH2zp16iguLk47duzQ8OHDFRoaql9//fWu7jMyMlJJSUnW5dSpU3d1fwAAALg3ShR1BxwdHVWzZk1JUpMmTbRz507Nnj1bvXr1UmpqqhITE23u3p47d05eXl6SJC8vL/3444827WU9TSGrTk6cnJzk5ORUyCMBAABAUSvyO7d/lZmZqZSUFDVp0kQODg5av369dd3Bgwd18uRJBQUFSZKCgoK0Z88eJSQkWOusXbtWrq6uqlev3j3vOwAAAIpWkd65jYyMVMeOHVW1alVdvnxZS5Ys0aZNm7R69Wq5ubkpLCxMo0ePVrly5eTq6qoRI0YoKChIDz30kCSpffv2qlevnvr166fXX39d8fHxmjBhgsLDw7kzCwAA8DdUpOE2ISFB/fv319mzZ+Xm5qbAwECtXr1a7dq1kyTNnDlTdnZ26tGjh1JSUhQSEqL33nvPur29vb1Wrlyp4cOHKygoSC4uLgoNDdXUqTyGCwAA4O+o2D3ntijwnFv8XfCcWwDA/eq+e84tAAAAcKcItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDQItwAAADANwi0AAABMg3ALAAAA0yDcAgAAwDSKNNxGRUWpWbNmKlOmjDw8PNS9e3cdPHjQpk6rVq1ksVhslmHDhtnUOXnypDp37qxSpUrJw8NDY8aMUXp6+r0cCgAAAIqBEkW5882bNys8PFzNmjVTenq6xo8fr/bt2+vXX3+Vi4uLtd7gwYM1depU6+tSpUpZ/52RkaHOnTvLy8tL27Zt09mzZ9W/f385ODho+vTp93Q8AAAAKFpFGm5XrVpl8zomJkYeHh7atWuXWrRoYS0vVaqUvLy8cmxjzZo1+vXXX7Vu3Tp5enqqUaNGevnllzVu3DhNnjxZjo6Od3UMAAAAKD6K1ZzbpKQkSVK5cuVsyhcvXqwKFSqoQYMGioyM1LVr16zrtm/froCAAHl6elrLQkJClJycrH379uW4n5SUFCUnJ9ssAAAAuP8V6Z3bm2VmZmrkyJF65JFH1KBBA2v5P//5T/n6+srb21u7d+/WuHHjdPDgQX322WeSpPj4eJtgK8n6Oj4+Psd9RUVFacqUKXdpJAAAACgqxSbchoeHa+/evdq6datN+ZAhQ6z/DggIUKVKldS2bVsdPXpUNWrUKNC+IiMjNXr0aOvr5ORk+fj4FKzjAAAAKDaKxbSEiIgIrVy5Uhs3blSVKlVuWbd58+aSpCNHjkiSvLy8dO7cOZs6Wa9zm6fr5OQkV1dXmwUAAAD3vyINt4ZhKCIiQp9//rk2bNggPz+/224TFxcnSapUqZIkKSgoSHv27FFCQoK1ztq1a+Xq6qp69erdlX4DAACgeCrSaQnh4eFasmSJvvzyS5UpU8Y6R9bNzU3Ozs46evSolixZok6dOql8+fLavXu3Ro0apRYtWigwMFCS1L59e9WrV0/9+vXT66+/rvj4eE2YMEHh4eFycnIqyuEBAADgHivSO7dz585VUlKSWrVqpUqVKlmXZcuWSZIcHR21bt06tW/fXnXr1tULL7ygHj166Ouvv7a2YW9vr5UrV8re3l5BQUF6+umn1b9/f5vn4gIAAODvoUjv3BqGccv1Pj4+2rx5823b8fX11TfffFNY3QIAAMB9qlh8oAwAAAAoDIRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmAbhFgAAAKZBuAUAAIBpEG4BAABgGoRbAAAAmEaRhtuoqCg1a9ZMZcqUkYeHh7p3766DBw/a1Llx44bCw8NVvnx5lS5dWj169NC5c+ds6pw8eVKdO3dWqVKl5OHhoTFjxig9Pf1eDgUAAADFQJGG282bNys8PFw//PCD1q5dq7S0NLVv315Xr1611hk1apS+/vprLV++XJs3b9aZM2f05JNPWtdnZGSoc+fOSk1N1bZt2xQbG6uYmBhNnDixKIYEAACAImQxDMMo6k5kOX/+vDw8PLR582a1aNFCSUlJqlixopYsWaKePXtKkg4cOCB/f39t375dDz30kL799lt16dJFZ86ckaenpyRp3rx5GjdunM6fPy9HR8fb7jc5OVlubm5KSkqSq6troYylfa+phdIOUJjWLOM/fQCA+1Ne81qxmnOblJQkSSpXrpwkadeuXUpLS1NwcLC1Tt26dVW1alVt375dkrR9+3YFBARYg60khYSEKDk5Wfv27ctxPykpKUpOTrZZAAAAcP8rULitXr26Lly4kK08MTFR1atXL1BHMjMzNXLkSD3yyCNq0KCBJCk+Pl6Ojo5yd3e3qevp6an4+HhrnZuDbdb6rHU5iYqKkpubm3Xx8fEpUJ8BAABQvBQo3B4/flwZGRnZylNSUnT69OkCdSQ8PFx79+7V0qVLC7R9fkRGRiopKcm6nDp16q7vEwAAAHdfifxU/uqrr6z/Xr16tdzc3KyvMzIytH79elWrVi3fnYiIiNDKlSu1ZcsWValSxVru5eWl1NRUJSYm2ty9PXfunLy8vKx1fvzxR5v2sp6mkFXnr5ycnOTk5JTvfgIAAKB4y1e47d69uyTJYrEoNDTUZp2Dg4OqVaumN998M8/tGYahESNG6PPPP9emTZvk5+dns75JkyZycHDQ+vXr1aNHD0nSwYMHdfLkSQUFBUmSgoKC9MorryghIUEeHh6SpLVr18rV1VX16tXLz/AAAABwn8tXuM3MzJQk+fn5aefOnapQocId7Tw8PFxLlizRl19+qTJlyljnyLq5ucnZ2Vlubm4KCwvT6NGjVa5cObm6umrEiBEKCgrSQw89JElq37696tWrp379+un1119XfHy8JkyYoPDwcO7OAgAA/M3kK9xmOXbsWKHsfO7cuZKkVq1a2ZRHR0drwIABkqSZM2fKzs5OPXr0UEpKikJCQvTee+9Z69rb22vlypUaPny4goKC5OLiotDQUE2dyqO4AAAA/m4K/Jzb9evXa/369UpISLDe0c2ycOHCQuncvcJzbvF3wXNuAQD3q7zmtQLduZ0yZYqmTp2qpk2bqlKlSrJYLAXuKAAAAFBYChRu582bp5iYGPXr16+w+wMAAAAUWIGec5uamqqHH364sPsCAAAA3JEChdtBgwZpyZIlhd0XAAAA4I4UaFrCjRs3NH/+fK1bt06BgYFycHCwWf/WW28VSucAAACA/ChQuN29e7caNWokSdq7d6/NOj5cBgAAgKJSoHC7cePGwu4HAAAAcMcKNOcWAAAAKI4KdOe2devWt5x+sGHDhgJ3CAAAACioAoXbrPm2WdLS0hQXF6e9e/cqNDS0MPoFAAAA5FuBwu3MmTNzLJ88ebKuXLlyRx0CAAAACqpQ59w+/fTTWrhwYWE2CQAAAORZoYbb7du3q2TJkoXZJAAAAJBnBZqW8OSTT9q8NgxDZ8+e1U8//aSXXnqpUDoGAAAA5FeBwq2bm5vNazs7O9WpU0dTp05V+/btC6VjAAAAQH4VKNxGR0cXdj8AAACAO1agcJtl165d2r9/vySpfv36aty4caF0CgAAACiIAoXbhIQE9e7dW5s2bZK7u7skKTExUa1bt9bSpUtVsWLFwuwjAAAAkCcFelrCiBEjdPnyZe3bt08XL17UxYsXtXfvXiUnJ+u5554r7D4CAAAAeVKgO7erVq3SunXr5O/vby2rV6+e3n33XT5QBgAAgCJToDu3mZmZcnBwyFbu4OCgzMzMO+4UAAAAUBAFCrdt2rTR888/rzNnzljLTp8+rVGjRqlt27aF1jkAAAAgPwoUbt955x0lJyerWrVqqlGjhmrUqCE/Pz8lJydrzpw5hd1HAAAAIE8KNOfWx8dHP//8s9atW6cDBw5Ikvz9/RUcHFyonQMAAADyI193bjds2KB69eopOTlZFotF7dq104gRIzRixAg1a9ZM9evX13fffXe3+goAAADcUr7C7axZszR48GC5urpmW+fm5qahQ4fqrbfeKrTOAQAAAPmRr3D7yy+/qEOHDrmub9++vXbt2nXHnQIAAAAKIl/h9ty5czk+AixLiRIldP78+TvuFAAAAFAQ+Qq3lStX1t69e3Ndv3v3blWqVOmOOwUAAAAURL7CbadOnfTSSy/pxo0b2dZdv35dkyZNUpcuXQqtcwAAAEB+5OtRYBMmTNBnn32m2rVrKyIiQnXq1JEkHThwQO+++64yMjL073//+650FAAAALidfIVbT09Pbdu2TcOHD1dkZKQMw5AkWSwWhYSE6N1335Wnp+dd6SgAAABwO/n+EgdfX1998803unTpko4cOSLDMFSrVi2VLVv2bvQPAAAAyLMCfUOZJJUtW1bNmjUrzL4AAAAAdyRfHygDAAAAijPCLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANIo03G7ZskVdu3aVt7e3LBaLvvjiC5v1AwYMkMVisVk6dOhgU+fixYvq27evXF1d5e7urrCwMF25cuUejgIAAADFRZGG26tXr6phw4Z69913c63ToUMHnT171rp88sknNuv79u2rffv2ae3atVq5cqW2bNmiIUOG3O2uAwAAoBgq8NfvFoaOHTuqY8eOt6zj5OQkLy+vHNft379fq1at0s6dO9W0aVNJ0pw5c9SpUye98cYb8vb2LvQ+AwAAoPgq9nNuN23aJA8PD9WpU0fDhw/XhQsXrOu2b98ud3d3a7CVpODgYNnZ2WnHjh25tpmSkqLk5GSbBQAAAPe/Yh1uO3TooI8++kjr16/Xa6+9ps2bN6tjx47KyMiQJMXHx8vDw8NmmxIlSqhcuXKKj4/Ptd2oqCi5ublZFx8fn7s6DgAAANwbRTot4XZ69+5t/XdAQIACAwNVo0YNbdq0SW3bti1wu5GRkRo9erT1dXJyMgEXAADABIr1ndu/ql69uipUqKAjR45Ikry8vJSQkGBTJz09XRcvXsx1nq705zxeV1dXmwUAAAD3v/sq3P7++++6cOGCKlWqJEkKCgpSYmKidu3aZa2zYcMGZWZmqnnz5kXVTQAAABSRIp2WcOXKFetdWEk6duyY4uLiVK5cOZUrV05TpkxRjx495OXlpaNHj2rs2LGqWbOmQkJCJEn+/v7q0KGDBg8erHnz5iktLU0RERHq3bs3T0oAAAD4GyrSO7c//fSTGjdurMaNG0uSRo8ercaNG2vixImyt7fX7t271a1bN9WuXVthYWFq0qSJvvvuOzk5OVnbWLx4serWrau2bduqU6dOevTRRzV//vyiGhIAAACKUJHeuW3VqpUMw8h1/erVq2/bRrly5bRkyZLC7BYAAADuU/fVnFsAAADgVgi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANAi3AAAAMA3CLQAAAEyDcAsAAADTINwCAADANIo03G7ZskVdu3aVt7e3LBaLvvjiC5v1hmFo4sSJqlSpkpydnRUcHKzDhw/b1Ll48aL69u0rV1dXubu7KywsTFeuXLmHowAAAEBxUaTh9urVq2rYsKHefffdHNe//vrrevvttzVv3jzt2LFDLi4uCgkJ0Y0bN6x1+vbtq3379mnt2rVauXKltmzZoiFDhtyrIQAAAKAYKVGUO+/YsaM6duyY4zrDMDRr1ixNmDBBjz/+uCTpo48+kqenp7744gv17t1b+/fv16pVq7Rz5041bdpUkjRnzhx16tRJb7zxhry9ve/ZWAAAAFD0iu2c22PHjik+Pl7BwcHWMjc3NzVv3lzbt2+XJG3fvl3u7u7WYCtJwcHBsrOz044dO3JtOyUlRcnJyTYLAAAA7n/FNtzGx8dLkjw9PW3KPT09revi4+Pl4eFhs75EiRIqV66ctU5OoqKi5ObmZl18fHwKufcAAAAoCsU23N5NkZGRSkpKsi6nTp0q6i4BAACgEBTbcOvl5SVJOnfunE35uXPnrOu8vLyUkJBgsz49PV0XL1601smJk5OTXF1dbRYAAADc/4ptuPXz85OXl5fWr19vLUtOTtaOHTsUFBQkSQoKClJiYqJ27dplrbNhwwZlZmaqefPm97zPAAAAKFpF+rSEK1eu6MiRI9bXx44dU1xcnMqVK6eqVatq5MiRmjZtmmrVqiU/Pz+99NJL8vb2Vvfu3SVJ/v7+6tChgwYPHqx58+YpLS1NERER6t27N09KAAAA+Bsq0nD7008/qXXr1tbXo0ePliSFhoYqJiZGY8eO1dWrVzVkyBAlJibq0Ucf1apVq1SyZEnrNosXL1ZERITatm0rOzs79ejRQ2+//fY9HwsAAACKnsUwDKOoO1HUkpOT5ebmpqSkpEKbf9u+19RCaQcoTGuWTSzqLgAAUCB5zWvFds4tAAAAkF+EWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJgG4RYAAACmQbgFAACAaRBuAQAAYBqEWwAAAJhGsQ63kydPlsVisVnq1q1rXX/jxg2Fh4erfPnyKl26tHr06KFz584VYY8BAABQlIp1uJWk+vXr6+zZs9Zl69at1nWjRo3S119/reXLl2vz5s06c+aMnnzyySLsLQAAAIpSiaLuwO2UKFFCXl5e2cqTkpK0YMECLVmyRG3atJEkRUdHy9/fXz/88IMeeuihe91VAAAAFLFif+f28OHD8vb2VvXq1dW3b1+dPHlSkrRr1y6lpaUpODjYWrdu3bqqWrWqtm/ffss2U1JSlJycbLMAAADg/lesw23z5s0VExOjVatWae7cuTp27Jgee+wxXb58WfHx8XJ0dJS7u7vNNp6enoqPj79lu1FRUXJzc7MuPj4+d3EUAAAAuFeK9bSEjh07Wv8dGBio5s2by9fXV59++qmcnZ0L3G5kZKRGjx5tfZ2cnEzABQAAMIFifef2r9zd3VW7dm0dOXJEXl5eSk1NVWJiok2dc+fO5ThH92ZOTk5ydXW1WQAAAHD/u6/C7ZUrV3T06FFVqlRJTZo0kYODg9avX29df/DgQZ08eVJBQUFF2EsAAAAUlWI9LeHFF19U165d5evrqzNnzmjSpEmyt7dXnz595ObmprCwMI0ePVrlypWTq6urRowYoaCgIJ6UAAAA8DdVrMPt77//rj59+ujChQuqWLGiHn30Uf3www+qWLGiJGnmzJmys7NTjx49lJKSopCQEL333ntF3GsAAAAUFYthGEZRd6KoJScny83NTUlJSYU2/7Z9r6mF0g5QmNYsm1jUXQAAoEDymtfuqzm3AAAAwK0QbgEAAGAahFsAAACYBuEWAAAApkG4BQAAgGkQbgEAAGAahFsAAACYBuEWAAAApkG4BQAAgGkQbgEAAGAahFsAAACYBuEWAAAApkG4BQAAgGkQbgEAAGAahFsAAACYBuEWAO4Dr776qiwWi0aOHHnLeomJiQoPD1elSpXk5OSk2rVr65tvvrGuX7x4sXx8fFS2bFmNHj3aZtvjx4+rdu3aSk5OvhtDAIB7okRRdwAAcGs7d+7U+++/r8DAwFvWS01NVbt27eTh4aEVK1aocuXKOnHihNzd3SVJf/zxhwYNGqSYmBhVr15dnTt3Vps2bdSlSxdJ0rPPPqtXX31Vrq6ud3tIAHDXEG4BoBi7cuWK+vbtqw8++EDTpk27Zd2FCxfq4sWL2rZtmxwcHCRJ1apVs67/7bff5Obmpl69ekmSWrdurf3796tLly765JNP5ODgoCeffPKujQUA7gWmJQBAMRYeHq7OnTsrODj4tnW/+uorBQUFKTw8XJ6enmrQoIGmT5+ujIwMSVKtWrV07do1/e9//9PFixe1c+dOBQYG6tKlS3rppZf0zjvv3O3hAMBdx51bACimli5dqp9//lk7d+7MU/3ffvtNGzZsUN++ffXNN9/oyJEjevbZZ5WWlqZJkyapbNmyio2NVf/+/XX9+nX1799fISEhCgsLU0REhI4dO6Zu3bopLS1NkydPVs+ePe/yCAGg8BFuAaAYOnXqlJ5//nmtXbtWJUuWzNM2mZmZ8vDw0Pz582Vvb68mTZro9OnTmjFjhiZNmiRJeuKJJ/TEE09Yt9m8ebN2796tOXPmqGbNmvrkk0/k5eWlBx98UC1atJCHh8ddGR8A3C2EWwAohnbt2qWEhAQ98MAD1rKMjAxt2bJF77zzjlJSUmRvb2+zTaVKleTg4GBT7u/vr/j4eKWmpsrR0dGmfkpKip599lktWrRIR44cUXp6ulq2bClJql27tnbs2KGuXbvexVECQOFjzi0AFENt27bVnj17FBcXZ12aNm2qvn37Ki4uLluwlaRHHnlER44cUWZmprXs0KFDqlSpUrZgK0nTpk1Thw4d9MADDygjI0Pp6enWdWlpada5ugBwP+HOLQAUQ2XKlFGDBg1sylxcXFS+fHlref/+/VW5cmVFRUVJkoYPH6533nlHzz//vEaMGKHDhw9r+vTpeu6557K1/+uvv2rZsmX63//+J0mqW7eu7OzstGDBAnl5eenAgQNq1qzZXR4lABQ+wi0A3KdOnjwpO7v/+wOcj4+PVq9erVGjRikwMFCVK1fW888/r3HjxtlsZxiGhgwZorfeeksuLi6SJGdnZ8XExCg8PFwpKSl65513VLly5Xs6HgAoDBbDMIyi7kRRS05Olpubm5KSkgrt4eXte00tlHaAwrRm2cSi7gIAAAWS17zGnFsAAACYBuEWAAAApsGcWwDFSqNpk4u6C4CNuAmTi7oLAPKBO7cAAAAwDcItAAAATINwCwAAANMg3AIAANN59913Va1aNZUsWVLNmzfXjz/+mGvdzz77TE2bNpW7u7tcXFzUqFEjLVq0yKbOG2+8IQ8PD3l4eOjNN9+0Wbdjxw41adLE5lv+UHT4QBkAADCVZcuWafTo0Zo3b56aN2+uWbNmKSQkRAcPHpSHh0e2+uXKldO///1v1a1bV46Ojlq5cqUGDhwoDw8PhYSEaPfu3Zo4caJWrlwpwzDUpUsXtW/fXgEBAUpPT9ewYcM0f/58lShBrCoOuHMLAABM5a233tLgwYM1cOBA1atXT/PmzVOpUqW0cOHCHOu3atVKTzzxhPz9/VWjRg09//zzCgwM1NatWyVJBw4cUGBgoNq0aaO2bdsqMDBQBw4ckCTNmDFDLVq04OuqixHCLQAAMI3U1FTt2rVLwcHB1jI7OzsFBwdr+/btt93eMAytX79eBw8eVIsWLSRJAQEBOnTokE6ePKkTJ07o0KFDatCggY4eParo6GhNmzbtro0H+Ue4BQAApvHHH38oIyNDnp6eNuWenp6Kj4/PdbukpCSVLl1ajo6O6ty5s+bMmaN27dpJkvz9/TV9+nS1a9dO7du3V1RUlPz9/TV06FC9/vrrWr16tRo0aKDGjRtry5Ytd3V8uD0mhwAAgL+9MmXKKC4uTleuXNH69es1evRoVa9eXa1atZIkDRs2TMOGDbPWj42NVZkyZRQUFKQ6depo586d+v3339W7d28dO3ZMTk5ORTQSEG4BAIBpVKhQQfb29jp37pxN+blz5+Tl5ZXrdnZ2dqpZs6YkqVGjRtq/f7+ioqKs4fZmf/zxh6ZMmaItW7Zox44dql27tmrVqqVatWopLS1Nhw4dUkBAQKGOC3nHtAQAAGAajo6OatKkidavX28ty8zM1Pr16xUUFJTndjIzM5WSkpLjulGjRmnUqFGqUqWKMjIylJaWZl2Xnp6ujIyMgg8Ad4w7twAAwFRGjx6t0NBQNW3aVA8++KBmzZqlq1evauDAgZKk/v37q3LlyoqKipIkRUVFqWnTpqpRo4ZSUlL0zTffaNGiRZo7d262tteuXatDhw4pNjZWktSsWTMdOHBA3377rU6dOiV7e3vVqVPn3g0W2RBuAQCAqfTq1Uvnz5/XxIkTFR8fr0aNGmnVqlXWD5mdPHlSdnb/98frq1ev6tlnn9Xvv/8uZ2dn1a1bVx9//LF69epl0+7169cVERGhZcuWWbevUqWK5syZo4EDB8rJyUmxsbFydna+d4NFNhbDMIyi7kRhePfddzVjxgzFx8erYcOGmjNnjh588ME8bZucnCw3NzclJSXJ1dW1UPrTvtfUQmkHKExrlk0s6i7cVqNpk4u6C4CNuAmTi7oLAJT3vGaKObdZ30QyadIk/fzzz2rYsKFCQkKUkJBQ1F0DAADAPWSKcJvfbyIBAACAOd33c26zvokkMjLSWna7byJJSUmx+QRkUlKSpD9vdxeW9LQbhdYWUFgK8xy/WzJu5PzpZKCo3A/XjSSN2TS2qLsA2JjR6vVCbS/rWrzdjNr7Ptze6ptIsr73+a+ioqI0ZcqUbOU+Pj53pY9AceH2eVRRdwG477i98mpRdwG4L83X+3el3cuXL8vNzS3X9fd9uC2IyMhIjR492vo6MzNTFy9eVPny5WWxWIqwZ/ir5ORk+fj46NSpU4X2YT/g74BrB8g/rpvizTAMXb58Wd7e3resd9+H24J8E4mTk1O2r8Vzd3e/W11EIXB1deWNBigArh0g/7huiq9b3bHNct9/oKywvokEAAAA97/7/s6tdPtvIgEAAMDfgynC7e2+iQT3LycnJ02aNCnbNBIAt8a1A+Qf1405mOYbygAAAID7fs4tAAAAkIVwCwAAANMg3AIAAMA0CLcAAAAwDcItcjRgwABZLBZZLBY5ODjIz89PY8eO1Y0bN6x1stb/dVm6dKkkadOmTbJYLEpMTMxxH5MnT85x+7p160qSAgICNGzYsBy3XbRokZycnPTHH39Yy0JCQmRvb6+dO3fmOp5XX7X9Gs0vvvjC+q10N485p6VatWp5Pn4oPm7+uTo6OqpmzZqaOnWq0tPTrXXycu7kdC3ExMTc8pyxWCw6fvy4Jk+erEaNGlnbNAxDwcHBCgkJyba/9957T+7u7vr9999vOa6s6ytrqVixojp16qQ9e/bk2v+blw4dOljrVKtWTbNmzcpxP8ePH891bD/88IPefPNNlS1b1ua9Icu1a9fk6uqqt99+21r2ySefyN7eXuHh4bmOqX79+srIyLBZ5+7urpiYmGzjzmnZtGnTLY8dCkde3lclKSMjQzNnzlRAQIBKliypsmXLqmPHjvr+++9ttsu6nm4+NyUpMTExXz/Xm88FV1dXNWvWTF9++WWO+/rrUrJkSZvxde/ePdf9VKtWLcc2Xn31Ve3atct6jeSkbdu2evLJJ62vf//9dzk6OqpBgwa5jqlkyZI6ceKETXn37t01YMCAbOPOaZk8efItjpq5EG6Rqw4dOujs2bP67bffNHPmTL3//vuaNGmSTZ3o6GidPXvWZrnVm8Ff1a9fP9v2W7dulSSFhYVp6dKlun79erbtoqOj1a1bN1WoUEGSdPLkSW3btk0RERFauHBhjvsqWbKkXnvtNV26dCnH9bNnz7bpx1/Hl1Pwwf0h61w+fPiwXnjhBU2ePFkzZsyQlLdzJ7droVevXjbnTFBQkAYPHmxT5uPjk609i8Wi6Oho7dixQ++//3/fvX7s2DGNHTtWc+bMUZUqVfI0toMHD+rs2bNavXq1UlJS1LlzZ6WmpubY/5uXTz75JK+HT5K0bt26bG00adJE/fr109WrV/XZZ59l22bFihVKTU3V008/bS1bsGCBxo4dq08++STHQCxJv/32mz766KMc1z388MM2fXjqqaeyje/hhx/O19hQcLd7XzUMQ71799bUqVP1/PPPa//+/dq0aZN8fHzUqlUrffHFFzb1S5QooXXr1mnjxo131K+s9+6ffvpJjzzyiHr27JntP36urq7Zzum/hsfbmTp1arY2RowYoSZNmqhhw4Y5vqccP35cGzduVFhYmLUsJiZGTz31lJKTk7Vjx44c92WxWDRx4sRc+3JzH2bNmpVtfC+++GK+xnY/I9wiV05OTvLy8pKPj4+6d++u4OBgrV271qaOu7u7vLy8bJab/+d7OyVKlMi2fVZgffrpp3X9+nX95z//sdnm2LFj2rRpk80bQ3R0tLp06aLhw4frk08+yTEQBwcHy8vLS1FRUTn2xc3NzaYffx1fxYoV8zwuFC9Z57Kvr6+GDx+u4OBgffXVV5Lydu7kdi04OzvbnDOOjo4qVaqUTZm9vX2OffLx8dHs2bP14osv6tixYzIMQ2FhYWrfvr369euX57F5eHjIy8tLDzzwgEaOHKlTp07pwIEDOfb/5qVs2bL5OIJS+fLls7Xh4OAgDw8Pde3aNcdf4gsXLlT37t1Vrlw5SX9eu9u2bdO//vUv1a5dO8dALEkjRozQpEmTlJKSkm2do6OjTR+cnZ2zjc/R0TFfY0PB3e599dNPP9WKFSv00UcfadCgQfLz81PDhg01f/58devWTYMGDdLVq1et9V1cXPTMM8/oX//61x31K+u9u3bt2nr55ZeVnp6eLTBbLJZs53R+n49fpkyZbG24uLhI+vMGzbJly3Tt2jWbbWJiYlSpUiXrHWrDMBQdHa1+/frpn//8pxYsWJDjviIiIvTxxx9r7969Oa6/uQ9ubm7Zxle6dOl8je1+RrhFnuzdu1fbtm27p780KlSooMcffzzbL82YmBhVqVJF7du3l/R/bwxPP/206tatq5o1a2rFihXZ2rO3t9f06dM1Z86c2/7JF+bm7Oys1NTUPJ87NyvMayE0NFRt27bVM888o3feeUd79+61uZObH0lJSdYpQfc63IWFhWnDhg02d71+++03bdmyJdt/Qjt37iw3Nzc9/fTTuf4SHzlypNLT0zVnzpy73nfcmdu9ry5ZskS1a9dW165ds6174YUXdOHChWw3TSZPnqw9e/bc9lrMi/T0dOt5dq+vi759+yolJcVmHIZhKDY2VgMGDLD+x3fjxo26du2agoOD9fTTT2vp0qU2gT/LI488oi5dutxx8P87INwiVytXrlTp0qVVsmRJBQQEKCEhQWPGjLGp06dPH5UuXdpmOXnyZJ73sWfPnmzb3zzPNiwsTJs2bdKxY8ck/d8bQ2hoqOzs/jx9161bp2vXrlnnL97ql+YTTzyhRo0aZZtegb8HwzC0bt06rV69Wm3atMnzuZOXa6Gg5s+fr71792rkyJGaP39+vv9CUKVKFZUuXVru7u5asmSJunXrZp23/tf+37xMnz49X/t5+OGHs7WRJSQkRN7e3oqOjraWxcTEyMfHR23btpUkZWZmKiYmxjpFoXfv3tq6dav12r5ZqVKlNGnSJEVFRSkpKSlf/cS9d6v31UOHDsnf3z/H7bLKDx06ZFPu7e2t559/Xv/+979t5sbnR9bvJicnJ40aNUrVqlXTU089ZVMnKSkp2zndsWPHfO1n3Lhx2dr47rvvJEnlypXTE088YXODZuPGjTp+/LgGDhxoLVuwYIF69+4te3t7NWjQQNWrV9fy5ctz3F9UVJRWrVpl3QdyRrhFrlq3bq24uDjt2LFDoaGhGjhwoHr06GFTZ+bMmYqLi7NZvL2987yPOnXqZNt+6tSp1vXt2rVTlSpVrL80169fr5MnT9q8MSxcuFC9evVSiRJ/fpt0nz599P333+vo0aM57vO1115TbGys9u/fn+d+4v52czjt2LGjevXqpcmTJ+f53MnLtVBQHh4eGjp0qPz9/fM1Xz3Ld999p127dikmJka1a9fWvHnzstXJ6v/NS24f1szNsmXLsrWRxd7eXqGhoYqJiZFhGMrMzFRsbKwGDhxo/U/o2rVrdfXqVXXq1EnSn3+ZadeuXa7znMPCwlS+fHm99tpr+eonisat3lcL8kWo48aN0/nz53M9P24n63fTt99+q3r16unDDz+0To/JUqZMmWzn9Icffpiv/YwZMyZbG02bNrWuf+aZZ7Rlyxbre8rChQvVsmVL1axZU9KfH5b77LPPbOal3+oGTb169dS/f3/u3t5GiaLuAIovFxcX6wW4cOFCNWzYUAsWLLD5M6OXl5e1TkFkfXo9N3Z2dhowYIBiY2M1efJkRUdHq3Xr1qpevbok6eLFi/r888+VlpamuXPnWrfLyMjQwoUL9corr2Rrs0WLFgoJCVFkZKT1U6Ywt9atW2vu3LlydHSUt7e3SpQoka9zJy/Xwp0oUaKENWDnl5+fn9zd3VWnTh0lJCSoV69e2rJli02dm/tfUD4+Prds45lnnlFUVJQ2bNigzMxMnTp1KtvdqYsXL8rZ2dlalpmZqd27d2vKlCnWEJylRIkSeuWVVzRgwABFRETcUd9x9+X2vlq7du1cbyRkldeuXTvbOnd3d0VGRmrKlCnq0qVLvvuT9bupZs2aio6OVqdOnfTrr7/Kw8PDWsfOzu6Or4sKFSrcso22bduqatWqiomJ0ZgxY/TZZ5/ZTD1asmSJbty4oebNm1vLsv6DeOjQoRyPzZQpU1S7du1sH8bD/+HOLfLEzs5O48eP14QJE3L8wM3dNHDgQJ06dUqfffaZPv/8c5tAsXjxYlWpUkW//PKLzf+c33zzTcXExGR7nFCWV199VV9//bW2b99+r4aBIpQV7qpWrWoNkQU9d4ryWrid8PBw7d27V59//vk933eNGjXUsmVLLVy4UNHR0QoODpavr68k6cKFC/ryyy+1dOlSm2P9v//9T5cuXdKaNWtybPMf//iH6tevrylTptzLoaCAcnpf7d27tw4fPqyvv/46W/0333xT5cuXV7t27XJsb8SIEbKzs9Ps2bPvqF8PPvigmjRpkuPNjrvNzs5OAwcOVGxsrJYsWSJHR0f17NnTun7BggV64YUXbK6LX375RY899liud619fHwUERGh8ePH5/o+9XfHnVvk2T/+8Q+NGTNG7777rvWRIomJiYqPj7epV6ZMGeunRaU/59WWKVPG+tpisahhw4aS/pzs/9ftLRaLzSdW/fz81KZNGw0ZMkROTk42zwZcsGCBevbsme3ZgD4+PoqMjNSqVavUuXPnbGMJCAhQ3759bZ6/ib+Xgp47Us7Xwu1cv37d5k/50p/XSo0aNQrU/5yUKlVKgwcP1qRJk9S9e3frs0ZTUlKyXWclSpSwPplEkk6fPp2tf1nhVPozoP61DXd3d5uno4SFhWnw4MGS/pxzm2XRokUqX768nnrqKZvnn0pSp06dtGDBgmzPNs3y6quv5vg8YBQ/Ob2v9u7dW8uXL1doaKhmzJihtm3bKjk5We+++66++uorLV++3Ob3xc1KliypKVOm5PhM5PwaOXKknnjiCY0dO1aVK1eW9Ocd0r+e09KfU4Wy/pKQlJSU7booX7689RF/ly9fztZGqVKl5Orqan09cOBATZ06VePHj1efPn2sf72Ii4vTzz//rMWLF2ebJ9+nTx9NnTpV06ZNy/GvOpGRkfrggw907Ngx9erVK59H42/AAHIQGhpqPP7449nKo6KijIoVKxpXrlwxJOW4REVFGYZhGBs3bsxxvb29vWEYhjFp0qQc1zs5OWXb75IlSwxJxrPPPmst++mnnwxJxo8//pjjGDp27Gg88cQTuY7n2LFjhqOjo5HbZSDJ+Pzzz293qFDM5fSzv9NzxzBsr4UsLVu2NJ5//vlsdXM719u2bWtd37Bhw3yNK+v6unTpkk35yZMnjRIlShjLli2z9j+nfdepU8e6ja+vb451Fi1aZBw7dizXa/2TTz6x2fe1a9cMNzc3o1y5csaNGzes5QEBATbX7s2WLVtmODo6GufPn891TO3btzckGdHR0dm2z+3ng7svr++raWlpxowZM4z69esbjo6OhqurqxESEmJs3brVZtvo6GjDzc3Npiw9Pd2oV6+eIcnYuHFjnvqV03t3ZmamUbduXWP48OHWfeV2Xp89e9Y6vpzWh4WFGYaR+3UzdOjQbH3KOodvfs+JiIgw6tWrl+MYzp49a9jZ2RlffvllrmOaPn26IckIDQ3Ntn1Ox/LvxGIYBZjpDQAAABRDzLkFAACAaRBuAaAY6tixY7bnZxb0GbWAWUyfPj3X6yK/z6iFeTEtAQCKodOnT+f6NIZy5cple2Yn8Hdw8eJFXbx4Mcd1zs7O1g+L4e+NcAsAAADTYFoCAAAATINwCwAAANMg3AIAAMA0CLcAAAAwDcItABQjmzZtksViUWJiYlF3BQDuS4RbAChEAwYMkMVikcVikYODg/z8/DR27FjduHGjqLsGAH8LJYq6AwBgNh06dFB0dLTS0tK0a9cuhYaGymKx6LXXXivqrgGA6XHnFgAKmZOTk7y8vOTj46Pu3bsrODhYa9eulSRlZmYqKipKfn5+cnZ2VsOGDbVixYpbtrd161Y99thjcnZ2lo+Pj5577jldvXpVkjR+/Hg1b9482zYNGzbU1KlTJUk7d+5Uu3btVKFCBbm5ually5b6+eefbepbLBZ9+OGHeuKJJ1SqVCnVqlVLX331lU2dffv2qUuXLnJ1dVWZMmX02GOP6ejRo9b1H374ofz9/VWyZEnVrVtX7733Xv4PHgDcIcItANxFe/fu1bZt2+To6ChJioqK0kcffaR58+Zp3759GjVqlJ5++mlt3rw5x+2PHj2qDh06qEePHtq9e7eWLVumrVu3KiIiQpLUt29f/fjjjzYhc9++fdq9e7f++c9/SpIuX76s0NBQbd26VT/88INq1aqlTp066fLlyzb7mjJlip566int3r1bnTp1Ut++fa3fBnX69Gm1aNFCTk5O2rBhg3bt2qVnnnlG6enpkqTFixdr4sSJeuWVV7R//35Nnz5dL730kmJjYwv3gALA7RgAgEITGhpq2NvbGy4uLoaTk5MhybCzszNWrFhh3LhxwyhVqpSxbds2m23CwsKMPn36GIZhGBs3bjQkGZcuXbKuGzJkiE397777zrCzszOuX79uGIZhNGzY0Jg6dap1fWRkpNG8efNc+5iRkWGUKVPG+Prrr61lkowJEyZYX1+5csWQZHz77bfWNv38/IzU1NQc26xRo4axZMkSm7KXX37ZCAoKyrUfAHA3MOcWAApZ69atNXfuXF29elUzZ85UiRIl1KNHD+3bt0/Xrl1Tu3btbOqnpqaqcePGObb1yy+/aPfu3Vq8eLG1zDAMZWZm6tixY/L391ffvn21cOFCvfTSSzIMQ5988olGjx5trX/u3DlNmDBBmzZtUkJCgjIyMnTt2jWdPHnSZl+BgYHWf7u4uMjV1VUJCQmSpLi4OD322GNycHDI1serV6/q6NGjCgsL0+DBg63l6enpcnNzy8eRA4A7R7gFgELm4uKimjVrSpIWLlyohg0basGCBWrQoIEk6b///a8qV65ss42Tk1OObV25ckVDhw7Vc889l21d1apVJUl9+vTRuHHj9PPPP+v69es6deqUevXqZa0XGhqqCxcuaPbs2fL19ZWTk5OCgoKUmppq095fg6vFYlFmZqYkydnZOdfxXrlyRZL0wQcfZJv/a29vn+t2AHA3EG4B4C6ys7PT+PHjNXr0aB06dEhOTk46efKkWrZsmaftH3jgAf3666/WsJyTKlWqqGXLllq8eLGuX7+udu3aycPDw7r++++/13vvvadOnTpJkk6dOqU//vgjX+MIDAxUbGys0tLSsoVgT09PeXt767ffflPfvn3z1S4AFDY+UAYAd9k//vEP2dvb6/3339eLL76oUaNGKTY2VkePHtXPP/+sOXPm5PrBq3Hjxmnbtm2KiIhQXFycDh8+rC+//NL6gbIsffv21dKlS7V8+fJsAbNWrVpatGiR9u/frx07dqhv3763vBObk4iICCUnJ6t379766aefdPjwYS1atEgHDx6U9OeH0aKiovT222/r0KFD2rNnj6Kjo/XWW2/laz8AcKcItwBwl5UoUUIRERF6/fXXFRkZqZdeeklRUVHy9/dXhw4d9N///ld+fn45bhsYGKjNmzfr0KFDeuyxx9S4cWNNnDhR3t7eNvV69uypCxcu6Nq1a+revbvNugULFujSpUt64IEH1K9fPz333HM2d3bzonz58tqwYYOuXLmili1bqkmTJvrggw+sd3EHDRqkDz/8UNHR0QoICFDLli0VExOT67gA4G6xGIZhFHUnAAAAgMLAnVsAAACYBuEWAAAApkG4BQAAgGkQbgEAAGAahFsAAACYBuEWAAAApkG4BQAAgGkQbgEAAGAahFsAAACYBuEWAAAApkG4BQAAgGn8P1bFkUnMzwPxAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate counts and percentages\n",
    "counts = df_eval_llama['relevance'].value_counts()\n",
    "print(counts)\n",
    "percentages = counts / counts.sum() * 100\n",
    "\n",
    "# Create bar plot\n",
    "plt.figure(figsize=(8, 5))\n",
    "ax = sns.countplot(data=df_eval, x='relevance', palette='viridis')\n",
    "\n",
    "# Add percentage labels on top of the bars\n",
    "for p in ax.patches:\n",
    "    height = p.get_height()\n",
    "    ax.annotate(f'{height / counts.sum() * 100:.1f}%', \n",
    "                (p.get_x() + p.get_width() / 2., height), \n",
    "                ha='center', va='bottom')\n",
    "\n",
    "plt.title('Distribution of Relevance For llama')\n",
    "plt.xlabel('Relevance')\n",
    "plt.ylabel('Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}