Created using Colaboratory

notebooks/09-Better_Embedding_Model.ipynb

@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyOuxoYC0mOLg3qm/0hreMFy",
+      "authorship_tag": "ABX9TyPOc0CzdBgKoadyg4eV8JWo",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -725,7 +725,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "QPJzr-I9XQ7l",
         "colab": {
@@ -787,13 +787,13 @@
         "import os\n",
         "\n",
         "# Set the \"OPENAI_API_KEY\" in the Python environment. Will be used by OpenAI client later.\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_OPENAI_KEY>\"\n",
-        "os.environ[\"CO_API_KEY\"] = \"<YOUR_COHERE_KEY>\""
+        "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
+        "os.environ[\"CO_API_KEY\"] = \"844npZDOg4G9mRqCiIBeJZCrvMuXHdS4lTBYrqoY\""
       ],
       "metadata": {
         "id": "riuXwpSPcvWC"
       },
-      "execution_count": 2,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -806,7 +806,7 @@
       "metadata": {
         "id": "jIEeZzqLbz0J"
       },
-      "execution_count": 3,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -828,7 +828,7 @@
       "metadata": {
         "id": "9oGT6crooSSj"
       },
-      "execution_count": 4,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -853,7 +853,7 @@
       "metadata": {
         "id": "SQP87lHczHKc"
       },
-      "execution_count": 5,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -867,7 +867,7 @@
       "metadata": {
         "id": "zAaGcYMJzHAN"
       },
-      "execution_count": 6,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -909,7 +909,7 @@
         "id": "wl_pbPvMlv1h",
         "outputId": "bc9a0415-a1fb-4e89-a2b4-165420106b34"
       },
-      "execution_count": 7,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -964,7 +964,7 @@
         },
         "outputId": "a8361aa6-522d-4def-e49b-ed08d9c8e7d1"
       },
-      "execution_count": 8,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -998,7 +998,7 @@
       "metadata": {
         "id": "YizvmXPejkJE"
       },
-      "execution_count": 9,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1015,6 +1015,8 @@
       "source": [
         "from llama_index.text_splitter import TokenTextSplitter\n",
         "\n",
+        "# Define the splitter object that split the text into segments with 512 tokens,\n",
+        "# with a 128 overlap between the segments.\n",
         "text_splitter = TokenTextSplitter(\n",
         "    separator=\" \", chunk_size=512, chunk_overlap=128\n",
         ")"
@@ -1022,7 +1024,7 @@
       "metadata": {
         "id": "9z3t70DGWsjO"
       },
-      "execution_count": 10,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1049,6 +1051,8 @@
         "from llama_index.embeddings.cohereai import CohereEmbedding\n",
         "from llama_index.ingestion import IngestionPipeline\n",
         "\n",
+        "# Create the pipeline to apply the transformation on each chunk,\n",
+        "# and store the transformed text in the chroma vector store.\n",
         "pipeline = IngestionPipeline(\n",
         "    transformations=[\n",
         "        text_splitter,\n",
@@ -1060,6 +1064,7 @@
         "    vector_store=vector_store\n",
         ")\n",
         "\n",
+        "# Run the transformation pipeline.\n",
         "nodes = pipeline.run(documents=documents, show_progress=True);"
       ],
       "metadata": {
@@ -1094,7 +1099,7 @@
         "id": "P9LDJ7o-Wsc-",
         "outputId": "cd49bff2-b0da-4722-8baa-6a07f1023b39"
       },
-      "execution_count": 11,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "display_data",
@@ -1167,7 +1172,7 @@
         "id": "mPGa85hM2P3P",
         "outputId": "9d7811ba-1e10-4098-b6eb-77a4e7d37457"
       },
-      "execution_count": 12,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -1193,7 +1198,7 @@
         "id": "jjnmscmq2cXK",
         "outputId": "5f6fa176-4e09-4cc7-bd17-8236b061ad17"
       },
-      "execution_count": 13,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -1210,6 +1215,7 @@
     {
       "cell_type": "code",
       "source": [
+        "# Compress the vector store directory to a zip file to be able to download and use later.\n",
         "!zip -r vectorstore_cohere.zip mini-llama-articles"
       ],
       "metadata": {
@@ -1219,7 +1225,7 @@
         "id": "hV9G0lSUJJSa",
         "outputId": "453a4ea3-dfda-4da1-ac29-929834c83b40"
       },
-      "execution_count": 33,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1245,10 +1251,19 @@
         "id": "OWaT6rL7ksp8"
       }
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "If you have already uploaded the zip file for the vector store checkpoint, please uncomment the code in the following cell block to extract its contents. After doing so, you will be able to load the dataset from local storage."
+      ],
+      "metadata": {
+        "id": "B4w8xP2Ggrvf"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
-        "!unzip vectorstore_cohere.zip"
+        "# !unzip vectorstore_cohere.zip"
       ],
       "metadata": {
         "id": "EF-wobGAJRgL"
@@ -1267,7 +1282,7 @@
       "metadata": {
         "id": "mXi56KTXk2sp"
       },
-      "execution_count": 14,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1275,11 +1290,14 @@
       "source": [
         "from llama_index import ServiceContext\n",
         "\n",
+        "# Define the Cohere Embedding Model\n",
         "embed_model = CohereEmbedding(\n",
         "    model_name=\"embed-english-v3.0\",\n",
         "    input_type=\"search_query\",\n",
         ")\n",
         "\n",
+        "# Define the ServiceCotext object to tie the LLM for generating final answer,\n",
+        "# and the embedding model to help with retrieving related nodes.\n",
         "service_context = ServiceContext.from_defaults(\n",
         "    llm=llm, embed_model=embed_model\n",
         ")"
@@ -1287,21 +1305,21 @@
       "metadata": {
         "id": "9l0PaY230syE"
       },
-      "execution_count": 37,
+      "execution_count": null,
       "outputs": []
     },
     {
       "cell_type": "code",
       "source": [
-        "# Create your index\n",
         "from llama_index import VectorStoreIndex\n",
         "\n",
+        "# Create the index based on the vector store.\n",
         "index = VectorStoreIndex.from_vector_store(vector_store, service_context=service_context)"
       ],
       "metadata": {
         "id": "jKXURvLtkuTS"
       },
-      "execution_count": 38,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1325,7 +1343,7 @@
       "metadata": {
         "id": "b0gue7cyctt1"
       },
-      "execution_count": 29,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1341,7 +1359,7 @@
         "id": "VKK3jMprctre",
         "outputId": "cb85d598-d1bc-49e9-818f-c7bbde465864"
       },
-      "execution_count": 30,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -1361,6 +1379,7 @@
     {
       "cell_type": "code",
       "source": [
+        "# Show the retrieved nodes\n",
         "for src in res.source_nodes:\n",
         "  print(\"Node ID\\t\", src.node_id)\n",
         "  print(\"Title\\t\", src.metadata['title'])\n",
@@ -1375,7 +1394,7 @@
         "id": "465dH4yQc7Ct",
         "outputId": "3d2b3ce2-7705-41bb-80e3-4fe6b390dcef"
       },
-      "execution_count": 31,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1410,6 +1429,9 @@
         "from llama_index.evaluation import generate_question_context_pairs\n",
         "from llama_index.llms import OpenAI\n",
         "\n",
+        "# Create questions for each segment. These questions will be used to\n",
+        "# assess whether the retriever can accurately identify and return the\n",
+        "# corresponding segment when queried.\n",
         "llm = OpenAI(model=\"gpt-3.5-turbo\")\n",
         "rag_eval_dataset = generate_question_context_pairs(\n",
         "    nodes,\n",
@@ -1417,7 +1439,7 @@
         "    num_questions_per_chunk=1\n",
         ")\n",
         "\n",
-        "# We can save the dataset as a json file for later use.\n",
+        "# We can save the evaluation dataset as a json file for later use.\n",
         "rag_eval_dataset.save_json(\"./rag_eval_dataset_cohere.json\")"
       ],
       "metadata": {
@@ -1427,7 +1449,7 @@
         },
         "outputId": "85b0765e-5a42-4f60-ccff-fc4bc688f65a"
       },
-      "execution_count": 32,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1438,20 +1460,29 @@
         }
       ]
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "If you have uploaded the generated question JSON file, please uncomment the code in the next cell block. This will avoid the need to generate the questions manually, saving you time and effort."
+      ],
+      "metadata": {
+        "id": "998nNEGYhKhu"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
-        "from llama_index.finetuning.embeddings.common import (\n",
-        "    EmbeddingQAFinetuneDataset,\n",
-        ")\n",
-        "rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\n",
-        "    \"./rag_eval_dataset_cohere.json\"\n",
-        ")"
+        "# from llama_index.finetuning.embeddings.common import (\n",
+        "#     EmbeddingQAFinetuneDataset,\n",
+        "# )\n",
+        "# rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\n",
+        "#     \"./rag_eval_dataset_cohere.json\"\n",
+        "# )"
       ],
       "metadata": {
         "id": "3sA1K84U254o"
       },
-      "execution_count": 34,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1459,6 +1490,7 @@
       "source": [
         "import pandas as pd\n",
         "\n",
+        "#  A simple function to show the evaluation result.\n",
         "def display_results_retriever(name, eval_results):\n",
         "    \"\"\"Display results from evaluate.\"\"\"\n",
         "\n",
@@ -1481,7 +1513,7 @@
       "metadata": {
         "id": "H7ubvcbk27vr"
       },
-      "execution_count": 35,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1505,7 +1537,7 @@
         "id": "uNLxDxoc2-Ac",
         "outputId": "8a2df94d-99b5-4aa4-a31e-b6c94256d1bb"
       },
-      "execution_count": 39,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1568,7 +1600,7 @@
         "id": "3ukkWC9R2_0J",
         "outputId": "d177c25d-a163-4b71-97f4-2af468737bbb"
       },
-      "execution_count": 40,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",