Created using Colaboratory

notebooks/02-Basic_RAG.ipynb

@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyMeAGO/FbVAX7iPUxTHf1EO",
+      "authorship_tag": "ABX9TyP70PdVay2ty6up4u2jHr1P",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -372,9 +372,18 @@
         "<a href=\"https://colab.research.google.com/github/towardsai/ai-tutor-rag-system/blob/main/notebooks/02-Basic_RAG.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Install Packages and Setup Variables"
+      ],
+      "metadata": {
+        "id": "4Tw3tvMs6R-Y"
+      }
+    },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -409,12 +418,13 @@
       "source": [
         "import os\n",
         "\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
+        "# Set the \"OPENAI_API_KEY\" in the Python environment. Will be used by OpenAI client later.\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_OPENAI_KEY>\""
       ],
       "metadata": {
         "id": "MYvUA6CF2Le6"
       },
-      "execution_count": 49,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -427,7 +437,7 @@
       "metadata": {
         "id": "0ViVXXIqXBai"
       },
-      "execution_count": 50,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -442,7 +452,7 @@
     {
       "cell_type": "markdown",
       "source": [
-        "### Download Paul Graham Essay (JSON)"
+        "## Download Paul Graham Essay (JSON)"
       ],
       "metadata": {
         "id": "5JpI7GiZ--Gw"
@@ -461,7 +471,7 @@
         "id": "p6NEJT9S2OoH",
         "outputId": "47da0417-d8cb-4b2c-90a4-e6e0e08a8325"
       },
-      "execution_count": 52,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -496,7 +506,7 @@
     {
       "cell_type": "markdown",
       "source": [
-        "### Read JSON"
+        "## Read File"
       ],
       "metadata": {
         "id": "oYDd03Qn_clh"
@@ -507,18 +517,11 @@
       "source": [
         "import json\n",
         "\n",
+        "# Load the file as a JSON\n",
         "with open('./mini-dataset.json', 'r') as file:\n",
-        "    data = json.load(file)"
-      ],
-      "metadata": {
-        "id": "XjW10Cnz_DVZ"
-      },
-      "execution_count": 53,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
+        "    data = json.load(file)\n",
+        "\n",
+        "# The number of chunks in the essay dataset.\n",
         "len( data['chunks'] )"
       ],
       "metadata": {
@@ -528,7 +531,7 @@
         "id": "UcQ7Ge_XCuXa",
         "outputId": "19f545a3-835b-43ec-8183-0ecf7f291d24"
       },
-      "execution_count": 54,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -547,17 +550,9 @@
       "source": [
         "import pandas as pd\n",
         "\n",
-        "df = pd.DataFrame(data['chunks'])"
-      ],
-      "metadata": {
-        "id": "YNk1hxGVDqTu"
-      },
-      "execution_count": 55,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
+        "# Convert the JSON list to a Pandas Dataframe\n",
+        "df = pd.DataFrame(data['chunks'])\n",
+        "\n",
         "df.keys()"
       ],
       "metadata": {
@@ -567,7 +562,7 @@
         "id": "JKdFSOb0NXjx",
         "outputId": "a3196605-aade-4d69-e858-daf222e6c7ba"
       },
-      "execution_count": 56,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -584,7 +579,7 @@
     {
       "cell_type": "markdown",
       "source": [
-        "### Apply Embedding"
+        "# Generate Embedding"
       ],
       "metadata": {
         "id": "21pFDgNdW9rO"
@@ -597,6 +592,7 @@
         "\n",
         "client = OpenAI()\n",
         "\n",
+        "# Defining a function that converts a text to embedding vector using OpenAI's Ada model.\n",
         "def get_embedding(text):\n",
         "  try:\n",
         "    # Remove newlines\n",
@@ -611,7 +607,7 @@
       "metadata": {
         "id": "AfS9w9eQAKyu"
       },
-      "execution_count": 57,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -619,11 +615,13 @@
       "source": [
         "from tqdm.notebook import tqdm\n",
         "\n",
+        "# Generate embedding\n",
         "if not load_embedding:\n",
         "  print(\"Generating embeddings...\")\n",
         "  for index, row in tqdm( df.iterrows() ):\n",
         "    row['embedding'] = get_embedding( row['text'] )\n",
         "\n",
+        "# Or, load the embedding from the file.\n",
         "else:\n",
         "  print(\"Loaded the embedding file.\")\n",
         "  with open('./mini-dataset_with_embedding.json', 'r') as file:\n",
@@ -651,7 +649,7 @@
         "id": "qC6aeFr3Rmi2",
         "outputId": "02208934-78fe-4a77-e375-aed39da6b678"
       },
-      "execution_count": 59,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -688,28 +686,10 @@
     {
       "cell_type": "code",
       "source": [
-        "QUESTION = \"How many parameters LLaMA2 model has?\""
-      ],
-      "metadata": {
-        "id": "mHt4bZjYRaiY"
-      },
-      "execution_count": 30,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "QUESTION_emb = get_embedding( QUESTION )"
-      ],
-      "metadata": {
-        "id": "OMQG6DtaVKGl"
-      },
-      "execution_count": 33,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
+        "# Define the user question, and convert it to embedding.\n",
+        "QUESTION = \"How many parameters LLaMA2 model has?\"\n",
+        "QUESTION_emb = get_embedding( QUESTION )\n",
+        "\n",
         "len( QUESTION_emb )"
       ],
       "metadata": {
@@ -719,7 +699,7 @@
         "id": "xGTa7cqCX97q",
         "outputId": "6935f145-1858-4959-833e-db2da1d49342"
       },
-      "execution_count": 34,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -736,12 +716,21 @@
     {
       "cell_type": "markdown",
       "source": [
-        "# Calculate Cosine Similarities"
+        "# Test Cosine Similarity"
       ],
       "metadata": {
         "id": "BXNzNWrJYWhU"
       }
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Calculating the similarity of embedding representations can help us to find pieces of text that are close to each other. In the following sample you see how the Cosine Similarity metric can identify which sentence could be a possible answer for the given user question. Obviously, the unrelated answer will score lower."
+      ],
+      "metadata": {
+        "id": "Vxaq-FgLIhIj"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
@@ -751,7 +740,7 @@
       "metadata": {
         "id": "LqDWcPd4b-ZI"
       },
-      "execution_count": 35,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -771,7 +760,7 @@
         "id": "OI00eN86YZKB",
         "outputId": "3c75fdaa-098e-467c-daa8-41e87dda0738"
       },
-      "execution_count": 36,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -784,20 +773,21 @@
       ]
     },
     {
-      "cell_type": "code",
+      "cell_type": "markdown",
       "source": [
-        "cosine_similarities = cosine_similarity( [QUESTION_emb], df['embedding'].tolist() )"
+        "# Calculate Cosine Similarities"
       ],
       "metadata": {
-        "id": "iYzyEU6-bwSz"
-      },
-      "execution_count": 37,
-      "outputs": []
+        "id": "kdJlEtaaJC4I"
+      }
     },
     {
       "cell_type": "code",
       "source": [
-        "cosine_similarities"
+        "# The similarity between the questions and each part of the essay.\n",
+        "cosine_similarities = cosine_similarity( [QUESTION_emb], df['embedding'].tolist() )\n",
+        "\n",
+        "print( cosine_similarities )"
       ],
       "metadata": {
         "colab": {
@@ -806,7 +796,7 @@
         "id": "PNPN7OAXemmH",
         "outputId": "5b76bf4f-bd0c-4023-9144-a9e251b843c9"
       },
-      "execution_count": 38,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -829,32 +819,14 @@
       "source": [
         "import numpy as np\n",
         "\n",
-        "# Sort the scores\n",
-        "highest_index = np.argmax( cosine_similarities )"
-      ],
-      "metadata": {
-        "id": "g0cBfePFaayw"
-      },
-      "execution_count": 39,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
         "number_of_chunks_to_retrieve = 3\n",
         "\n",
-        "indices = np.argsort(cosine_similarities[0])[::-1][:number_of_chunks_to_retrieve]"
-      ],
-      "metadata": {
-        "id": "vBbHJ7uihfKO"
-      },
-      "execution_count": 40,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "indices"
+        "# Sort the scores\n",
+        "highest_index = np.argmax( cosine_similarities )\n",
+        "\n",
+        "# Pick the N highest scored chunks\n",
+        "indices = np.argsort(cosine_similarities[0])[::-1][:number_of_chunks_to_retrieve]\n",
+        "print( indices )"
       ],
       "metadata": {
         "colab": {
@@ -863,7 +835,7 @@
         "id": "1-XI1_7mhlw4",
         "outputId": "c91bd5bc-c547-4760-d821-9f13fd564392"
       },
-      "execution_count": 41,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -880,7 +852,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Get the highest scored pieces of text\n",
+        "# Look at the highest scored retrieved pieces of text\n",
         "for idx, item in enumerate( df.text[indices] ):\n",
         "  print(f\"> Chunk {idx+1}\")\n",
         "  print(item)\n",
@@ -893,7 +865,7 @@
         "id": "JPmhCb9kfB0w",
         "outputId": "ee0f9bc2-3353-4ae8-b292-2356169b0d68"
       },
-      "execution_count": 42,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -924,18 +896,20 @@
     {
       "cell_type": "code",
       "source": [
+        "# Use the OpenAI API to answer the questions based on the retrieved pieces of text.\n",
         "try:\n",
-        "    # Formulating the system prompt\n",
+        "    # Formulating the system prompt and condition the model to answer only AI-related questions.\n",
         "    system_prompt = (\n",
         "      \"You are an assistant and expert in answering questions from a chunks of content. \"\n",
         "      \"Only answer AI-related question, else say that you cannot answer this question.\"\n",
         "    )\n",
         "\n",
-        "    # Combining the system prompt with the user's question\n",
+        "    # Create a user prompt with the user's question\n",
         "    prompt = (\n",
         "      \"Read the following informations that might contain the context you require to answer the question. You can use the informations starting from the <START_OF_CONTEXT> tag and end with the <END_OF_CONTEXT> tag. Here is the content:\\n\\n<START_OF_CONTEXT>\\n{}\\n<END_OF_CONTEXT>\\n\\n\"\n",
         "      \"Please provide an informative and accurate answer to the following question based on the avaiable context. Be concise and take your time. \\nQuestion: {}\\nAnswer:\"\n",
         "    )\n",
+        "    # Add the retrieved pieces of text to the prompt.\n",
         "    prompt = prompt.format( \"\".join( df.text[indices] ), QUESTION )\n",
         "\n",
         "    # Call the OpenAI API\n",
@@ -957,7 +931,7 @@
       "metadata": {
         "id": "MXRdzta5kJ3V"
       },
-      "execution_count": 43,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -972,7 +946,7 @@
         "id": "9tBvJ8oMucha",
         "outputId": "f473068b-0edc-424f-bb11-f7cfea76dc81"
       },
-      "execution_count": 44,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -992,6 +966,15 @@
         "id": "pW-BNCAC2JzE"
       }
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Test the OpenAI API to answer the same question without the addition of retrieved documents. Basically, the LLM will use its knowledge to answer the question."
+      ],
+      "metadata": {
+        "id": "tr5zXEGIMwJu"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
@@ -1022,7 +1005,7 @@
       "metadata": {
         "id": "RuyXjzZyuecE"
       },
-      "execution_count": 31,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1037,7 +1020,7 @@
         "id": "YAy34tPTzGbh",
         "outputId": "54041329-dd5f-4cdd-db38-f1440ae77181"
       },
-      "execution_count": 32,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",