Created using Colab

notebooks/06-Evaluate_RAG.ipynb

@@ -21,11 +21,61 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": 1,
       "metadata": {
-        "id": "QPJzr-I9XQ7l"
+        "id": "QPJzr-I9XQ7l",
+        "outputId": "71591538-a161-4a0a-e2c4-057bd2de6941",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m28.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m46.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m52.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m64.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━���━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m145.0/145.0 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.8/130.8 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m26.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m55.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m42.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
+          ]
+        }
+      ],
       "source": [
         "!pip install -q llama-index==0.10.37 openai==1.30.1 tiktoken==0.7.0 chromadb==0.5.0 llama-index-vector-stores-chroma==0.1.7"
       ]
@@ -41,7 +91,7 @@
         "import os\n",
         "\n",
         "# Set the \"OPENAI_API_KEY\" in the Python environment. Will be used by OpenAI client later.\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"[YOUR_OPENAI_KEY]\""
+        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
       ]
     },
     {
@@ -86,7 +136,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": 5,
       "metadata": {
         "id": "zAaGcYMJzHAN"
       },
@@ -127,21 +177,21 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 8,
+      "execution_count": 6,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
         "id": "fQtpDvUzKNzI",
-        "outputId": "da1c5652-403c-418e-93ba-4749983c1cfa"
+        "outputId": "833814c6-f3f7-4812-d030-6e9d10b86566"
       },
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "--2024-06-11 16:47:13--  https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv\n",
-            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.110.133, 185.199.111.133, ...\n",
+            "--2024-06-26 15:43:09--  https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv\n",
+            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
             "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
             "HTTP request sent, awaiting response... 200 OK\n",
             "Length: 173646 (170K) [text/plain]\n",
@@ -149,7 +199,7 @@
             "\n",
             "mini-llama-articles 100%[===================>] 169.58K  --.-KB/s    in 0.03s   \n",
             "\n",
-            "2024-06-11 16:47:14 (4.95 MB/s) - ‘mini-llama-articles.csv’ saved [173646/173646]\n",
+            "2024-06-26 15:43:09 (4.78 MB/s) - ‘mini-llama-articles.csv’ saved [173646/173646]\n",
             "\n"
           ]
         }
@@ -169,13 +219,13 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 9,
+      "execution_count": 7,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
         "id": "_WER5lt0N7c5",
-        "outputId": "2e404b1b-a8f8-4b0d-c5ad-b4aef649b3f2"
+        "outputId": "17dbc7d6-9750-4748-93b6-e21c2ec90ce1"
       },
       "outputs": [
         {
@@ -186,7 +236,7 @@
             ]
           },
           "metadata": {},
-          "execution_count": 9
+          "execution_count": 7
         }
       ],
       "source": [
@@ -219,7 +269,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": 8,
       "metadata": {
         "id": "lFvW_886dxKX"
       },
@@ -242,7 +292,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 11,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -278,7 +328,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 12,
+      "execution_count": 9,
       "metadata": {
         "id": "STACTMUR1z9N"
       },
@@ -301,38 +351,38 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 13,
+      "execution_count": 10,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
           "height": 81,
           "referenced_widgets": [
-            "de111a67b56f43c7b8c62f5ae989eaae",
-            "d43aba64df1741c289de2d04934fc29f",
-            "0465399af85c4bfa8ee6ac541833ad0e",
-            "e7dd26ddf6434335bf3dd8d9ad3a1dfb",
-            "eda4aaa92b6542d5877628e00c9f86b9",
-            "1d78d8024aa049e99acafcb55a229a6b",
-            "8c9cd46c635f43d9aa7014fe2caf66d9",
-            "cd1d4f99e3e441af8a905ed8337c61e2",
-            "334a92afaf284b09b873f70f632198ce",
-            "dca0b364c2f14bcf8c4f53ca8ac36d70",
-            "427b40bae46143e5ba8a79bef1510216",
-            "767d542c637e4fcb9812bf2a641d7667",
-            "6f3642aa394449a8946b2c1fe3de03e8",
-            "6794ed529b9a49d8bd828b8ee942bc5c",
-            "01df7b933c8f43e2a9135c56f471ebfe",
-            "11a9b1b3c9964d04968d7e67a1fbc5af",
-            "f55eeedeb2fa4716a1bec1f4b1392230",
-            "b1704fd5ed8044e79c752de8d076e4a8",
-            "12862a5bac05411c93700050e99f379f",
-            "9a4c8a453357486286343cf5c817dc36",
-            "92be71ee85654aa8aaf57a385224ea2c",
-            "b528382dfa984f24a8ea7d762e427c8c"
+            "28c1cd3da3e94be4865255c868033798",
+            "580a479c2ae34923a59b2b9afd0ea8c9",
+            "32dc9fae0d6441cf906d9e45de311f60",
+            "8d71e6ced1e349ec88e9ad2feb312065",
+            "3c7b98951d114e2595ca7f6c6f464687",
+            "1b5e20be590b45c7bb5ddc90276ae493",
+            "7d481cc7244448d3953a6c37aee6ed62",
+            "2d3b076f01a84775b8944f8944d35c63",
+            "ce543bc180524b7e860860b3d0a204f2",
+            "9e1b63ba68354443a26a946c3099303c",
+            "2ec6143869d14484be1935294ddd3fc0",
+            "2b7d77e17cd647759f904291e5840507",
+            "fc469c78a6384be59f89ca9d039d3c25",
+            "ba569b44a8e74a948fa760f52cc4a5c8",
+            "8cfd1d9c93b24ead898b98dd5325377c",
+            "afa1f77f0fe140139a01702d5296f9e8",
+            "938dc498e428402db86667cddeb2cf07",
+            "0ec063511dcb4ebf80908497a6ac7b95",
+            "c119965f5f354bfba8e6ecdca2dd1138",
+            "425b798a2f9444578d81e991779e3b2d",
+            "b27784dfa6144372bcc222cf59e9aa05",
+            "37e7cb84cedd4644a0df421dff825faf"
           ]
         },
         "id": "CtdsIUQ81_hT",
-        "outputId": "0c6c1c2b-85b4-4afb-a51a-7957bcdc9e95"
+        "outputId": "c30bf148-3b0f-4f90-f49c-e97294e81f3f"
       },
       "outputs": [
         {
@@ -344,7 +394,7 @@
             "application/vnd.jupyter.widget-view+json": {
               "version_major": 2,
               "version_minor": 0,
-              "model_id": "de111a67b56f43c7b8c62f5ae989eaae"
+              "model_id": "28c1cd3da3e94be4865255c868033798"
             }
           },
           "metadata": {}
@@ -358,7 +408,7 @@
             "application/vnd.jupyter.widget-view+json": {
               "version_major": 2,
               "version_minor": 0,
-              "model_id": "767d542c637e4fcb9812bf2a641d7667"
+              "model_id": "2b7d77e17cd647759f904291e5840507"
             }
           },
           "metadata": {}
@@ -381,7 +431,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 14,
+      "execution_count": null,
       "metadata": {
         "collapsed": true,
         "colab": {
@@ -417,7 +467,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 15,
+      "execution_count": 11,
       "metadata": {
         "id": "HbT3-kRO4Qpt"
       },
@@ -431,7 +481,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 16,
+      "execution_count": 12,
       "metadata": {
         "id": "sb61DWU84bHP"
       },
@@ -442,7 +492,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 17,
+      "execution_count": 13,
       "metadata": {
         "id": "G32W2LMMCmnv"
       },
@@ -453,7 +503,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 18,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -483,7 +533,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 19,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -566,7 +616,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 20,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -599,7 +649,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 21,
+      "execution_count": 14,
       "metadata": {
         "id": "mNDd5i921Hww"
       },
@@ -634,7 +684,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 22,
+      "execution_count": null,
       "metadata": {
         "id": "eARSzx8I1Hww"
       },
@@ -664,7 +714,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 23,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -727,13 +777,13 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 24,
+      "execution_count": 16,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
         "id": "ckjE4fcD1Hwx",
-        "outputId": "0124b4cb-5629-4041-ac1a-d5863c37e576"
+        "outputId": "1123434a-180c-48a3-ec0a-f52aa4325026"
       },
       "outputs": [
         {
@@ -742,32 +792,38 @@
           "text": [
             "top_2 faithfulness_score: 1.0\n",
             "top_2 relevancy_score: 1.0\n",
+            "top_2 correctness: 0.75\n",
             "top_4 faithfulness_score: 1.0\n",
             "top_4 relevancy_score: 1.0\n",
+            "top_4 correctness: 0.85\n",
             "top_6 faithfulness_score: 1.0\n",
             "top_6 relevancy_score: 1.0\n",
+            "top_6 correctness: 0.8\n",
             "top_8 faithfulness_score: 1.0\n",
             "top_8 relevancy_score: 1.0\n",
-            "top_10 faithfulness_score: 0.95\n",
-            "top_10 relevancy_score: 0.95\n"
+            "top_8 correctness: 0.85\n",
+            "top_10 faithfulness_score: 1.0\n",
+            "top_10 relevancy_score: 1.0\n",
+            "top_10 correctness: 0.8\n"
           ]
         }
       ],
       "source": [
-        "from llama_index.core.evaluation import RelevancyEvaluator, FaithfulnessEvaluator, BatchEvalRunner\n",
+        "from llama_index.core.evaluation import RelevancyEvaluator, FaithfulnessEvaluator, CorrectnessEvaluator, BatchEvalRunner\n",
         "from llama_index.llms.openai import OpenAI\n",
         "\n",
         "llm_gpt4 = OpenAI(temperature=0, model=\"gpt-4o\")\n",
         "\n",
         "faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4)\n",
         "relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4)\n",
+        "correctness_evaluator = CorrectnessEvaluator(llm=llm_gpt4)\n",
         "\n",
         "# Run evaluation\n",
         "queries = list(rag_eval_dataset.queries.values())\n",
         "batch_eval_queries = queries[:20]\n",
         "\n",
         "runner = BatchEvalRunner(\n",
-        "{\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator},\n",
+        "{\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator, \"correctness\": correctness_evaluator},\n",
         "workers=32,\n",
         ")\n",
         "\n",
@@ -782,7 +838,10 @@
         "    print(f\"top_{i} faithfulness_score: {faithfulness_score}\")\n",
         "\n",
         "    relevancy_score = sum(result.passing for result in eval_results['relevancy']) / len(eval_results['relevancy'])\n",
-        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")\n"
+        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")\n",
+        "\n",
+        "    correctness = sum(result.passing for result in eval_results['correctness']) / len(eval_results['correctness'])\n",
+        "    print(f\"top_{i} correctness: {correctness}\")\n"
       ]
     },
     {
@@ -818,7 +877,7 @@
     },
     "widgets": {
       "application/vnd.jupyter.widget-state+json": {
-        "de111a67b56f43c7b8c62f5ae989eaae": {
+        "28c1cd3da3e94be4865255c868033798": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HBoxModel",
           "model_module_version": "1.5.0",
@@ -833,14 +892,14 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_d43aba64df1741c289de2d04934fc29f",
-              "IPY_MODEL_0465399af85c4bfa8ee6ac541833ad0e",
-              "IPY_MODEL_e7dd26ddf6434335bf3dd8d9ad3a1dfb"
+              "IPY_MODEL_580a479c2ae34923a59b2b9afd0ea8c9",
+              "IPY_MODEL_32dc9fae0d6441cf906d9e45de311f60",
+              "IPY_MODEL_8d71e6ced1e349ec88e9ad2feb312065"
             ],
-            "layout": "IPY_MODEL_eda4aaa92b6542d5877628e00c9f86b9"
+            "layout": "IPY_MODEL_3c7b98951d114e2595ca7f6c6f464687"
           }
         },
-        "d43aba64df1741c289de2d04934fc29f": {
+        "580a479c2ae34923a59b2b9afd0ea8c9": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HTMLModel",
           "model_module_version": "1.5.0",
@@ -855,13 +914,13 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_1d78d8024aa049e99acafcb55a229a6b",
+            "layout": "IPY_MODEL_1b5e20be590b45c7bb5ddc90276ae493",
             "placeholder": "​",
-            "style": "IPY_MODEL_8c9cd46c635f43d9aa7014fe2caf66d9",
+            "style": "IPY_MODEL_7d481cc7244448d3953a6c37aee6ed62",
             "value": "Parsing nodes: 100%"
           }
         },
-        "0465399af85c4bfa8ee6ac541833ad0e": {
+        "32dc9fae0d6441cf906d9e45de311f60": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "FloatProgressModel",
           "model_module_version": "1.5.0",
@@ -877,15 +936,15 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_cd1d4f99e3e441af8a905ed8337c61e2",
+            "layout": "IPY_MODEL_2d3b076f01a84775b8944f8944d35c63",
             "max": 14,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_334a92afaf284b09b873f70f632198ce",
+            "style": "IPY_MODEL_ce543bc180524b7e860860b3d0a204f2",
             "value": 14
           }
         },
-        "e7dd26ddf6434335bf3dd8d9ad3a1dfb": {
+        "8d71e6ced1e349ec88e9ad2feb312065": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HTMLModel",
           "model_module_version": "1.5.0",
@@ -900,13 +959,13 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_dca0b364c2f14bcf8c4f53ca8ac36d70",
+            "layout": "IPY_MODEL_9e1b63ba68354443a26a946c3099303c",
             "placeholder": "​",
-            "style": "IPY_MODEL_427b40bae46143e5ba8a79bef1510216",
-            "value": " 14/14 [00:00<00:00, 32.13it/s]"
+            "style": "IPY_MODEL_2ec6143869d14484be1935294ddd3fc0",
+            "value": " 14/14 [00:00<00:00, 12.73it/s]"
           }
         },
-        "eda4aaa92b6542d5877628e00c9f86b9": {
+        "3c7b98951d114e2595ca7f6c6f464687": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -958,7 +1017,7 @@
             "width": null
           }
         },
-        "1d78d8024aa049e99acafcb55a229a6b": {
+        "1b5e20be590b45c7bb5ddc90276ae493": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1010,7 +1069,7 @@
             "width": null
           }
         },
-        "8c9cd46c635f43d9aa7014fe2caf66d9": {
+        "7d481cc7244448d3953a6c37aee6ed62": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
@@ -1025,7 +1084,7 @@
             "description_width": ""
           }
         },
-        "cd1d4f99e3e441af8a905ed8337c61e2": {
+        "2d3b076f01a84775b8944f8944d35c63": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1077,7 +1136,7 @@
             "width": null
           }
         },
-        "334a92afaf284b09b873f70f632198ce": {
+        "ce543bc180524b7e860860b3d0a204f2": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "ProgressStyleModel",
           "model_module_version": "1.5.0",
@@ -1093,7 +1152,7 @@
             "description_width": ""
           }
         },
-        "dca0b364c2f14bcf8c4f53ca8ac36d70": {
+        "9e1b63ba68354443a26a946c3099303c": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1145,7 +1204,7 @@
             "width": null
           }
         },
-        "427b40bae46143e5ba8a79bef1510216": {
+        "2ec6143869d14484be1935294ddd3fc0": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
@@ -1160,7 +1219,7 @@
             "description_width": ""
           }
         },
-        "767d542c637e4fcb9812bf2a641d7667": {
+        "2b7d77e17cd647759f904291e5840507": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HBoxModel",
           "model_module_version": "1.5.0",
@@ -1175,14 +1234,14 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_6f3642aa394449a8946b2c1fe3de03e8",
-              "IPY_MODEL_6794ed529b9a49d8bd828b8ee942bc5c",
-              "IPY_MODEL_01df7b933c8f43e2a9135c56f471ebfe"
+              "IPY_MODEL_fc469c78a6384be59f89ca9d039d3c25",
+              "IPY_MODEL_ba569b44a8e74a948fa760f52cc4a5c8",
+              "IPY_MODEL_8cfd1d9c93b24ead898b98dd5325377c"
             ],
-            "layout": "IPY_MODEL_11a9b1b3c9964d04968d7e67a1fbc5af"
+            "layout": "IPY_MODEL_afa1f77f0fe140139a01702d5296f9e8"
           }
         },
-        "6f3642aa394449a8946b2c1fe3de03e8": {
+        "fc469c78a6384be59f89ca9d039d3c25": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HTMLModel",
           "model_module_version": "1.5.0",
@@ -1197,13 +1256,13 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_f55eeedeb2fa4716a1bec1f4b1392230",
+            "layout": "IPY_MODEL_938dc498e428402db86667cddeb2cf07",
             "placeholder": "​",
-            "style": "IPY_MODEL_b1704fd5ed8044e79c752de8d076e4a8",
+            "style": "IPY_MODEL_0ec063511dcb4ebf80908497a6ac7b95",
             "value": "Generating embeddings: 100%"
           }
         },
-        "6794ed529b9a49d8bd828b8ee942bc5c": {
+        "ba569b44a8e74a948fa760f52cc4a5c8": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "FloatProgressModel",
           "model_module_version": "1.5.0",
@@ -1219,15 +1278,15 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_12862a5bac05411c93700050e99f379f",
+            "layout": "IPY_MODEL_c119965f5f354bfba8e6ecdca2dd1138",
             "max": 108,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_9a4c8a453357486286343cf5c817dc36",
+            "style": "IPY_MODEL_425b798a2f9444578d81e991779e3b2d",
             "value": 108
           }
         },
-        "01df7b933c8f43e2a9135c56f471ebfe": {
+        "8cfd1d9c93b24ead898b98dd5325377c": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "HTMLModel",
           "model_module_version": "1.5.0",
@@ -1242,13 +1301,13 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_92be71ee85654aa8aaf57a385224ea2c",
+            "layout": "IPY_MODEL_b27784dfa6144372bcc222cf59e9aa05",
             "placeholder": "​",
-            "style": "IPY_MODEL_b528382dfa984f24a8ea7d762e427c8c",
-            "value": " 108/108 [00:01<00:00, 92.16it/s]"
+            "style": "IPY_MODEL_37e7cb84cedd4644a0df421dff825faf",
+            "value": " 108/108 [00:03<00:00, 33.40it/s]"
           }
         },
-        "11a9b1b3c9964d04968d7e67a1fbc5af": {
+        "afa1f77f0fe140139a01702d5296f9e8": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1300,7 +1359,7 @@
             "width": null
           }
         },
-        "f55eeedeb2fa4716a1bec1f4b1392230": {
+        "938dc498e428402db86667cddeb2cf07": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1352,7 +1411,7 @@
             "width": null
           }
         },
-        "b1704fd5ed8044e79c752de8d076e4a8": {
+        "0ec063511dcb4ebf80908497a6ac7b95": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
@@ -1367,7 +1426,7 @@
             "description_width": ""
           }
         },
-        "12862a5bac05411c93700050e99f379f": {
+        "c119965f5f354bfba8e6ecdca2dd1138": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1419,7 +1478,7 @@
             "width": null
           }
         },
-        "9a4c8a453357486286343cf5c817dc36": {
+        "425b798a2f9444578d81e991779e3b2d": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "ProgressStyleModel",
           "model_module_version": "1.5.0",
@@ -1435,7 +1494,7 @@
             "description_width": ""
           }
         },
-        "92be71ee85654aa8aaf57a385224ea2c": {
+        "b27784dfa6144372bcc222cf59e9aa05": {
           "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
@@ -1487,7 +1546,7 @@
             "width": null
           }
         },
-        "b528382dfa984f24a8ea7d762e427c8c": {
+        "37e7cb84cedd4644a0df421dff825faf": {
           "model_module": "@jupyter-widgets/controls",
           "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",