Created using Colab

notebooks/17-Using_LLMs_to_rank_chunks_as_the_Judge.ipynb

@@ -0,0 +1,830 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "authorship_tag": "ABX9TyMAX07BuQFwcBDYOOj62vXm",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/towardsai/ai-tutor-rag-system/blob/main/notebooks/17-Using_LLMs_to_rank_chunks_as_the_Judge.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Install Packages and Setup Variables"
+      ],
+      "metadata": {
+        "id": "0FbELaf7TrW7"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "Yubz8AanRRSW",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "2487c4fd-0fb5-4894-ffe6-c747f4adb952"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.7/226.7 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.1/211.1 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m508.6/508.6 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m23.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.3/81.3 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.6/97.6 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m28.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m55.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.8/91.8 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m56.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m47.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.1/60.1 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.1/106.1 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m698.9/698.9 kB\u001b[0m \u001b[31m55.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m80.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m35.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m92.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━���━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m78.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Building wheel for tinysegmenter (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Building wheel for feedfinder2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Building wheel for jieba3k (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Building wheel for sgmllib3k (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install -q llama-index==0.10.30 openai==1.12.0 tiktoken==0.6.0 newspaper3k==0.2.8 chromadb==0.4.21 llama-index-vector-stores-chroma==0.1.7"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "\n",
+        "# Set the \"OPENAI_API_KEY\" in the Python environment. Will be used by OpenAI client later.\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"[OPENAI_API_KEY]\""
+      ],
+      "metadata": {
+        "id": "xLXFuRW-TpUu"
+      },
+      "execution_count": 2,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Load a Model"
+      ],
+      "metadata": {
+        "id": "r6GCYYqqTuMc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.llms.openai import OpenAI\n",
+        "\n",
+        "llm = OpenAI(temperature=0.9, model=\"gpt-3.5-turbo\", max_tokens=512)"
+      ],
+      "metadata": {
+        "id": "pupJpdZaTu5m"
+      },
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Create a Vector Store"
+      ],
+      "metadata": {
+        "id": "gaKYO-KrTwsn"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import chromadb\n",
+        "\n",
+        "# create client and a new collection\n",
+        "# chromadb.EphemeralClient saves data in-memory.\n",
+        "chroma_client = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
+        "chroma_collection = chroma_client.create_collection(\"mini-llama-articles\")"
+      ],
+      "metadata": {
+        "id": "npCqCZSPZKR0"
+      },
+      "execution_count": 4,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+        "\n",
+        "# Define a storage context object using the created vector database.\n",
+        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+      ],
+      "metadata": {
+        "id": "dG9eKSVrZMs1"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Load the Dataset (CSV)"
+      ],
+      "metadata": {
+        "id": "HmiFENBdZMAk"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Download"
+      ],
+      "metadata": {
+        "id": "X-20isiTZRIa"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "The dataset includes several articles from the TowardsAI blog, which provide an in-depth explanation of the LLaMA2 model. Read the dataset as a long string."
+      ],
+      "metadata": {
+        "id": "-lWKX814ZURc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!wget https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv"
+      ],
+      "metadata": {
+        "id": "fmlEL849ZPrH",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "63039988-ab7a-4ecf-deb0-d9510628ecb8"
+      },
+      "execution_count": 6,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "--2024-04-30 18:37:36--  https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv\n",
+            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n",
+            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
+            "HTTP request sent, awaiting response... 200 OK\n",
+            "Length: 173646 (170K) [text/plain]\n",
+            "Saving to: ‘mini-llama-articles.csv’\n",
+            "\n",
+            "mini-llama-articles 100%[===================>] 169.58K  --.-KB/s    in 0.01s   \n",
+            "\n",
+            "2024-04-30 18:37:37 (11.3 MB/s) - ‘mini-llama-articles.csv’ saved [173646/173646]\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Read File"
+      ],
+      "metadata": {
+        "id": "r9PL_eiTZW7y"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import csv\n",
+        "\n",
+        "rows = []\n",
+        "\n",
+        "# Load the file as a JSON\n",
+        "with open(\"./mini-llama-articles.csv\", mode=\"r\", encoding=\"utf-8\") as file:\n",
+        "  csv_reader = csv.reader(file)\n",
+        "\n",
+        "  for idx, row in enumerate( csv_reader ):\n",
+        "    if idx == 0: continue; # Skip header row\n",
+        "    rows.append( row )\n",
+        "\n",
+        "# The number of characters in the dataset.\n",
+        "len( rows )"
+      ],
+      "metadata": {
+        "id": "x5IwXJi8ZQGh"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Convert to Document obj"
+      ],
+      "metadata": {
+        "id": "ktYUZzzSZaDW"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.core.schema import Document\n",
+        "\n",
+        "# Convert the chunks to Document objects so the LlamaIndex framework can process them.\n",
+        "documents = [Document(text=row[1], metadata={\"title\": row[0], \"url\": row[2], \"source_name\": row[3]}) for row in rows]"
+      ],
+      "metadata": {
+        "id": "oO10Q-UyZQEB"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Transforming"
+      ],
+      "metadata": {
+        "id": "0PnovZ0tZdAT"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.core.node_parser import TokenTextSplitter\n",
+        "\n",
+        "# Define the splitter object that split the text into segments with 512 tokens,\n",
+        "# with a 128 overlap between the segments.\n",
+        "text_splitter = TokenTextSplitter(\n",
+        "    separator=\" \", chunk_size=512, chunk_overlap=128\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "wzOQZH6VZQBm"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.core.extractors import (\n",
+        "    SummaryExtractor,\n",
+        "    QuestionsAnsweredExtractor,\n",
+        "    KeywordExtractor,\n",
+        ")\n",
+        "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+        "from llama_index.core.ingestion import IngestionPipeline\n",
+        "\n",
+        "# Create the pipeline to apply the transformation on each chunk,\n",
+        "# and store the transformed text in the chroma vector store.\n",
+        "pipeline = IngestionPipeline(\n",
+        "    transformations=[\n",
+        "        text_splitter,\n",
+        "        QuestionsAnsweredExtractor(questions=3, llm=llm),\n",
+        "        SummaryExtractor(summaries=[\"prev\", \"self\"], llm=llm),\n",
+        "        KeywordExtractor(keywords=10, llm=llm),\n",
+        "        OpenAIEmbedding(),\n",
+        "    ],\n",
+        "    vector_store=vector_store\n",
+        ")\n",
+        "\n",
+        "# Run the transformation pipeline.\n",
+        "nodes = pipeline.run(documents=documents, show_progress=True);"
+      ],
+      "metadata": {
+        "id": "l6UP7M_rZeXS"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "len( nodes )"
+      ],
+      "metadata": {
+        "id": "GcUUhs88ZeUs"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Compress the vector store directory to a zip file to be able to download and use later.\n",
+        "!zip -r vectorstore.zip mini-llama-articles"
+      ],
+      "metadata": {
+        "id": "B_P8Cil-ZeQM"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Load Indexes"
+      ],
+      "metadata": {
+        "id": "YSGHsZMMZj4E"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "If you have already uploaded the zip file for the vector store checkpoint, please uncomment the code in the following cell block to extract its contents. After doing so, you will be able to load the dataset from local storage."
+      ],
+      "metadata": {
+        "id": "J81Yvj0AZlvK"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# !unzip vectorstore.zip"
+      ],
+      "metadata": {
+        "id": "M8iaOOGyZeNp",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "6a117a0b-161a-4889-daf6-baf94ae00d2a"
+      },
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Archive:  vectorstore.zip\n",
+            "   creating: mini-llama-articles/\n",
+            "   creating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/\n",
+            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/data_level0.bin  \n",
+            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/header.bin  \n",
+            " extracting: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/link_lists.bin  \n",
+            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/length.bin  \n",
+            "  inflating: mini-llama-articles/chroma.sqlite3  \n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load the vector store from the local storage.\n",
+        "db = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
+        "chroma_collection = db.get_or_create_collection(\"mini-llama-articles\")\n",
+        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+      ],
+      "metadata": {
+        "id": "6tzS_EKPZeLS"
+      },
+      "execution_count": 8,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.core import VectorStoreIndex\n",
+        "\n",
+        "# Create the index based on the vector store.\n",
+        "index = VectorStoreIndex.from_vector_store(vector_store)"
+      ],
+      "metadata": {
+        "id": "0T6FL7J3ZrNK"
+      },
+      "execution_count": 9,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# RankGPT"
+      ],
+      "metadata": {
+        "id": "w2XBkzNwLle5"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from llama_index.core.postprocessor.rankGPT_rerank import RankGPTRerank\n",
+        "\n",
+        "rankGPT = RankGPTRerank(top_n=3, llm=OpenAI(model=\"gpt-3.5-turbo\"))"
+      ],
+      "metadata": {
+        "id": "_it2CxTtLmHT"
+      },
+      "execution_count": 10,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
+        "# and using a LLM to formulate the final answer.\n",
+        "# The `node_postprocessors` function will be applied to the retrieved nodes.\n",
+        "query_engine = index.as_query_engine(\n",
+        "    similarity_top_k=10,\n",
+        "    node_postprocessors=[rankGPT]\n",
+        ")\n",
+        "\n",
+        "res = query_engine.query(\"How many parameters LLaMA2 model has?\")"
+      ],
+      "metadata": {
+        "id": "YA3M9m9CL6AJ"
+      },
+      "execution_count": 11,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "res.response"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 53
+        },
+        "id": "wgyjv9e6MCVm",
+        "outputId": "70723d5e-9d16-4123-884b-0d65cd91a405"
+      },
+      "execution_count": 12,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'The Llama 2 model has four different parameter sizes: 7 billion, 13 billion, 34 billion, and 70 billion.'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 12
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Show the retrieved nodes\n",
+        "for src in res.source_nodes:\n",
+        "  print(\"Node ID\\t\", src.node_id)\n",
+        "  print(\"Title\\t\", src.metadata['title'])\n",
+        "  print(\"Text\\t\", src.text)\n",
+        "  print(\"Score\\t\", src.score)\n",
+        "  print(\"-_\"*20)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "wUhOlwWcMEUT",
+        "outputId": "eae3754b-5cb8-4c5d-d739-c42c9686006d"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Node ID\t d6f533e5-fef8-469c-a313-def19fd38efe\n",
+            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
+            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
+            "Score\t 0.7077337819711658\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
+            "Node ID\t 2f3b7c34-8fd0-4134-af38-ef1b77e32cd8\n",
+            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
+            "Text\t The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving an optimum balance that allows the model to be both helpful and safe is of utmost importance. To strike the right balance between helpfulness and safety, Meta employed two reward models - one for helpfulness and another for safety - to optimize the model's responses. The 34B parameter model has reported higher safety violations than other variants, possibly contributing to the delay in its release.  IV. Helpfulness Comparison: Llama 2 Outperforms Competitors Llama 2 emerges as a strong contender in the open-source language model arena, outperforming its competitors in most categories. The 70B parameter model outperforms all other open-source models, while the 7B and 34B models outshine Falcon in all categories and MPT in all categories except coding. Despite being smaller, Llam a2's performance rivals that of Chat GPT 3.5, a significantly larger closed-source model. While GPT 4 and PalM-2-L, with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering\n",
+            "Score\t 0.7025566634608498\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
+            "Node ID\t 1c7a8637-6f65-401e-be33-26886c828a34\n",
+            "Title\t Inside Code Llama: Meta AI's Entrance in the Code LLM Space\n",
+            "Text\t Inside Code Llama The release of Code Llama does not include a single model but three different variants, characterized by their parameter sizes of 7B, 13B, and 34B. Each of these models has been trained on an extensive pool of 500B tokens encompassing code and code-related information. Notably, the 7B and 13B base and instruct models have been endowed with fill-in-the-middle (FIM) competence, empowering them to seamlessly insert code into existing code structures. This attribute equips them to handle tasks like code completion right from the outset.The trio of models caters to distinct requisites concerning serving and latency. For instance, the 7B model boasts the ability to operate on a single GPU. While the 34B model stands out for yielding optimal outcomes and elevating coding assistance, the smaller 7B and 13B versions excel in speed, making them fitting for low-latency tasks such as real-time code completion. Meta AI's innovations further extend to two nuanced adaptations of Code Llama: Code Llama - Python and Code Llama - Instruct. Code Llama - Python is a specialized derivation, meticulously honed on a substantial volume of Python code spanning 100B tokens. Given Python's central role in code generation benchmarks and its significance within the AI community, this focused model augments utility.Code Llama - Instruct represents an alignment and refinement of Code Llama through instructional fine-tuning. This novel training approach entails furnishing the model with \"natural language instruction\" inputs paired with anticipated outputs. This strategic methodology enhances the model's capacity to grasp human expectations in prompts. For endeavors involving code generation, it is advised to opt for Code Llama - Instruct versions, as they have been calibrated to yield useful and secure natural language responses. Deep diving into the Code Llama training and fine-tuning, there are a few aspects that are worth highlighting 1) DatasetLlama's training rests on a meticulously curated dataset enriched with publicly available code, offering a near-duplicate-free landscape. The dataset consists of 500B tokens during the initial phase, starting from the 7B, 13B, and 34B\n",
+            "Score\t 0.6889534709415898\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Custom Postprocessor"
+      ],
+      "metadata": {
+        "id": "5mcAcZqhQluE"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## The `Judger` Function"
+      ],
+      "metadata": {
+        "id": "7v7vmJblQrN6"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "The following function will query GPT-4 to retrieve the top three nodes that has highest similarity to the asked question."
+      ],
+      "metadata": {
+        "id": "6k8IKlN9QvU7"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pydantic import BaseModel\n",
+        "from llama_index.llms.openai import OpenAI\n",
+        "from llama_index.core.prompts import PromptTemplate\n",
+        "\n",
+        "\n",
+        "def judger(nodes, query):\n",
+        "\n",
+        "  # The model's output template\n",
+        "  class OrderedNodes(BaseModel):\n",
+        "    \"\"\"A node with the id and assigned score.\"\"\"\n",
+        "    node_id: list\n",
+        "    score: list\n",
+        "\n",
+        "  # Prepare the nodes and wrap them in <NODE></NODE> identifier, as well as the query\n",
+        "  the_nodes=\"\"\n",
+        "  for idx, item in enumerate(nodes):\n",
+        "    the_nodes += f\"<NODE{idx+1}>\\nNode ID: {item.node_id}\\nText: {item.text}\\n</NODE{idx+1}>\\n\"\n",
+        "\n",
+        "  query = \"<QUERY>\\n{}\\n</QUERY>\".format(query)\n",
+        "\n",
+        "  # Define the prompt template\n",
+        "  prompt_tmpl = PromptTemplate(\n",
+        "    \"\"\"\n",
+        "    You receive a qurey along with a list of nodes' text and their ids. Your task is to assign score\n",
+        "    to each node based on its contextually closeness to the given query. The final output is each\n",
+        "    node id along with its proximity score.\n",
+        "    Here is the list of nodes:\n",
+        "    {nodes_list}\n",
+        "\n",
+        "    And the following is the query:\n",
+        "    {user_query}\n",
+        "\n",
+        "    Score each of the nodes based on their text and their relevancy to the provided query.\n",
+        "    The score must be a decimal number between 0 an 1 so we can rank them.\"\"\"\n",
+        "  )\n",
+        "\n",
+        "  # Define the an instance of GPT-4 and send the request\n",
+        "  llm = OpenAI(model=\"gpt-4\")\n",
+        "  ordered_nodes = llm.structured_predict(\n",
+        "    OrderedNodes, prompt_tmpl, nodes_list=the_nodes, user_query=query\n",
+        "  )\n",
+        "\n",
+        "  return ordered_nodes"
+      ],
+      "metadata": {
+        "id": "WhtJ1OeF9L3G"
+      },
+      "execution_count": 25,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Define Postprocessor"
+      ],
+      "metadata": {
+        "id": "Q5f1GrBKZprO"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "The following class will use the `judger` function to rank the nodes, and filter them based on the ranks."
+      ],
+      "metadata": {
+        "id": "yZujUJTvQ6Yu"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from typing import (\n",
+        "    List,\n",
+        "    Optional\n",
+        ")\n",
+        "from llama_index.core import QueryBundle\n",
+        "from llama_index.core.postprocessor.types import BaseNodePostprocessor\n",
+        "from llama_index.core.schema import NodeWithScore\n",
+        "\n",
+        "\n",
+        "class OpenaiAsJudgePostprocessor(BaseNodePostprocessor):\n",
+        "    def _postprocess_nodes(\n",
+        "        self, nodes: List[NodeWithScore], query_bundle: Optional[QueryBundle]\n",
+        "    ) -> List[NodeWithScore]:\n",
+        "\n",
+        "        r = judger(nodes, query_bundle)\n",
+        "\n",
+        "        node_ids = r.node_id\n",
+        "        scores = r.score\n",
+        "\n",
+        "        sorted_scores = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)\n",
+        "        top_three_nodes = [sorted_scores[i][0] for i in range(3)]\n",
+        "\n",
+        "        selected_nodes_id = [node_ids[item] for item in top_three_nodes]\n",
+        "\n",
+        "        final_nodes = []\n",
+        "        for item in nodes:\n",
+        "          if item.node_id in selected_nodes_id:\n",
+        "            final_nodes.append( item )\n",
+        "\n",
+        "        return final_nodes"
+      ],
+      "metadata": {
+        "id": "-QtyuC8fZun0"
+      },
+      "execution_count": 26,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "judge = OpenaiAsJudgePostprocessor()"
+      ],
+      "metadata": {
+        "id": "jk-lqYlYLipi"
+      },
+      "execution_count": 27,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Query Engine with Postprocessor"
+      ],
+      "metadata": {
+        "id": "cgtsvxR7SflP"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
+        "# and using a LLM to formulate the final answer.\n",
+        "# The `node_postprocessors` function will be applied to the retrieved nodes.\n",
+        "query_engine = index.as_query_engine(\n",
+        "    similarity_top_k=10,\n",
+        "    node_postprocessors=[judge]\n",
+        ")\n",
+        "\n",
+        "res = query_engine.query(\"How many parameters LLaMA2 model has?\")"
+      ],
+      "metadata": {
+        "id": "1Hh3RLCeLfXZ"
+      },
+      "execution_count": 28,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "res.response"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 53
+        },
+        "id": "zmZv0EIyF0wG",
+        "outputId": "7ff1b3bf-1b5f-4985-ea0d-3048d94c8da1"
+      },
+      "execution_count": 29,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 29
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Show the retrieved nodes\n",
+        "for src in res.source_nodes:\n",
+        "  print(\"Node ID\\t\", src.node_id)\n",
+        "  print(\"Title\\t\", src.metadata['title'])\n",
+        "  print(\"Text\\t\", src.text)\n",
+        "  print(\"Score\\t\", src.score)\n",
+        "  print(\"-_\"*20)"
+      ],
+      "metadata": {
+        "id": "bBMaG6yaZzjA",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "8a173ef7-e66f-4f9b-a979-c88a17028ef0"
+      },
+      "execution_count": 30,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Node ID\t d6f533e5-fef8-469c-a313-def19fd38efe\n",
+            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
+            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
+            "Score\t 0.7077337819711658\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
+            "Node ID\t 2f3b7c34-8fd0-4134-af38-ef1b77e32cd8\n",
+            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
+            "Text\t The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving an optimum balance that allows the model to be both helpful and safe is of utmost importance. To strike the right balance between helpfulness and safety, Meta employed two reward models - one for helpfulness and another for safety - to optimize the model's responses. The 34B parameter model has reported higher safety violations than other variants, possibly contributing to the delay in its release.  IV. Helpfulness Comparison: Llama 2 Outperforms Competitors Llama 2 emerges as a strong contender in the open-source language model arena, outperforming its competitors in most categories. The 70B parameter model outperforms all other open-source models, while the 7B and 34B models outshine Falcon in all categories and MPT in all categories except coding. Despite being smaller, Llam a2's performance rivals that of Chat GPT 3.5, a significantly larger closed-source model. While GPT 4 and PalM-2-L, with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering\n",
+            "Score\t 0.7025566634608498\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
+            "Node ID\t 021c859e-809b-49b8-8d0d-38cc326c1203\n",
+            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
+            "Text\t with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering strong competition to closed-source models.  V. Ghost Attention: Enhancing Conversational Continuity One unique feature in Llama 2 is Ghost Attention, which ensures continuity in conversations. This means that even after multiple interactions, the model remembers its initial instructions, ensuring more coherent and consistent responses throughout the conversation. This feature significantly enhances the user experience and makes Llama 2 a more reliable language model for interactive applications. In the example below, on the left, it forgets to use an emoji after a few conversations. On the right, with Ghost Attention, even after having many conversations, it will remember the context and continue to use emojis in its response.  VI. Temporal Capability: A Leap in Information Organization Meta reported a groundbreaking temporal capability, where the model organizes information based on time relevance. Each question posed to the model is associated with a date, and it responds accordingly by considering the event date before which the question becomes irrelevant. For example, if you ask the question, \"How long ago did Barack Obama become president?\", its only relevant after 2008. This temporal awareness allows Llama 2 to deliver more contextually accurate responses, enriching the user experience further.  VII. Open Questions and Future Outlook Meta's open-sourcing of Llama 2 represents a seismic shift, now offering developers and researchers commercial access to a leading language model. With Llama 2 outperforming MosaicML's current MPT models, all eyes are on how Databricks will respond. Can MosaicML's next MPT iteration beat Llama 2? Is it worthwhile to compete\n",
+            "Score\t 0.691486848320407\n",
+            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "J7sIPpFFTep3"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}