Upload laurent-restaurant-adaptation-mistral-7b-tuned.ipynb

laurent-restaurant-adaptation-mistral-7b-tuned.ipynb

@@ -0,0 +1,556 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Libraries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install bitsandbytes\n",
+    "! pip install einops\n",
+    "! pip install peft\n",
+    "! pip install datasets==2.14.6"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2.14.6\n",
+      "4.35.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Check the versions\n",
+    "import datasets\n",
+    "print(datasets.__version__)\n",
+    "import transformers\n",
+    "print(transformers.__version__)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Restaurant dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "bin C:\\Users\\Utilisateur\\anaconda3\\lib\\site-packages\\bitsandbytes\\libbitsandbytes_cuda117.dll\n"
+     ]
+    }
+   ],
+   "source": [
+    "import einops\n",
+    "import torch\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from datasets import load_dataset,Dataset\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments, Trainer, DataCollatorForLanguageModeling\n",
+    "from peft import LoraConfig,get_peft_model,AutoPeftModelForCausalLM"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Load of the dataset for domain adaptation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset0 = load_dataset(\"Argen7um/restrant-qa\")#.select(range(877))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Adaptation of the data for training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['text'],\n",
+       "    num_rows: 877\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "text = []\n",
+    "for i in range(877): \n",
+    "    text.append('At Laurent restaurant : '+ dataset0['train'][i]['Prompt'].split('[question]:')[1].replace(' [/INST]\\n',''))\n",
+    "\n",
+    "data_text = pd.DataFrame(columns = ['text'])\n",
+    "data_text['text'] = text\n",
+    "\n",
+    "dataset_text = Dataset.from_pandas(data_text)\n",
+    "dataset_text"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Check the distribution of the length of the rows (truncation impact ?)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[<Axes: title={'center': '0'}>]], dtype=object)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApJklEQVR4nO3df3RU5Z3H8c+ETAZSM4QQQpI1gfgTLRL5UdKsroWSAMGDRakrPzyLlAPFBVtJu0W6ogm2G1Zb1uqycNhVdA+ktPYIKioQQEDWEOVHFnFdJBTESgIFNhmSyDiQZ//gMHVMIJlwJ/ME3q9z5pD7PM+989zvueN8vHPnjssYYwQAAGCRmGhPAAAA4OsIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAFbw+/2aO3eu0tPT1a1bN+Xk5KisrCza0wIQJQQUAFZ46KGHtGjRIk2ePFm/+c1v1KVLF40ZM0bbt2+P9tQARIGLHwsEEG3vv/++cnJy9Mwzz+inP/2pJOnMmTPq37+/UlJS9N5770V5hgA6GmdQAETdH/7wB3Xp0kUzZswItnXt2lXTpk1TeXm5PvvssyjODkA0EFAARN2ePXt00003yev1hrQPHTpUklRZWRmFWQGIJgIKgKirrq5WWlpas/YLbUePHu3oKQGIMgIKgKj74osv5PF4mrV37do12A/g6kJAARB13bp1k9/vb9Z+5syZYD+AqwsBBUDUpaWlqbq6uln7hbb09PSOnhKAKCOgAIi622+/XZ988ol8Pl9Ie0VFRbAfwNWFgAIg6r7//e/r3LlzWrZsWbDN7/dr+fLlysnJUUZGRhRnByAaYqM9AQDIycnR/fffr3nz5un48eO64YYb9PLLL+vw4cN64YUXoj09AFHAnWQBWOHMmTOaP3++VqxYof/7v//TgAED9NRTT2nUqFHRnhqAKCCgAAAA63ANCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdTrljdqampp09OhRJSQkyOVyRXs6AACgDYwxOn36tNLT0xUTc+lzJJ0yoBw9epRbXwMA0El99tlnuvbaay85plMGlISEBEnnd9Dr9QbbA4GANmzYoJEjR8rtdkdrelcEaukcaukcaukcaukcatl2Pp9PGRkZwffxS+mUAeXCxzper7dZQImPj5fX6+UguUzU0jnU0jnU0jnU0jnUMnxtuTyDi2QBAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOuEHVC2bdumsWPHKj09XS6XS2vWrAnpd7lcLT6eeeaZ4Ji+ffs261+4cOFl7wwAALgyhB1QGhoalJ2drcWLF7fYX11dHfJ48cUX5XK5NH78+JBxCxYsCBn3yCOPtG8PAADAFSfsW90XFBSooKDgov2pqakhy6+99pqGDx+u6667LqQ9ISGh2VgAAAApwr/Fc+zYMb355pt6+eWXm/UtXLhQTz31lDIzMzVp0iTNmTNHsbEtT8fv98vv9weXfT6fpPO/fxAIBILtF/7+ahvah1o6h1o6h1o6h1o6h1q2XTg1imhAefnll5WQkKD77rsvpP1HP/qRBg0apKSkJL333nuaN2+eqqurtWjRoha3U1JSouLi4mbtGzZsUHx8fLP2srIyZ3YA1NJB1NI51NI51NI51LJ1jY2NbR7rMsaY9j6Ry+XS6tWrNW7cuBb7+/Xrp/z8fD3//POX3M6LL76oH/7wh6qvr5fH42nW39IZlIyMDJ04caLZrxmXlZUpPz+fX5S8TNTSOdTSOdTSOdTSOdSy7Xw+n5KTk1VXVxfy/t2SiJ1Beffdd7V//3797ne/a3VsTk6Ozp49q8OHD+vmm29u1u/xeFoMLm63u8WD4WLtCF8ka9n3sTfbve7hhXc7OJOOwXHpHGrpHGrpHGrZunDqE7H7oLzwwgsaPHiwsrOzWx1bWVmpmJgYpaSkRGo6AACgEwn7DEp9fb2qqqqCy4cOHVJlZaWSkpKUmZkp6fwpnFdeeUW//vWvm61fXl6uiooKDR8+XAkJCSovL9ecOXP04IMPqkePHpexKwAA4EoRdkDZuXOnhg8fHlwuLCyUJE2ZMkUvvfSSJGnVqlUyxmjixInN1vd4PFq1apWKiork9/uVlZWlOXPmBLcDAAAQdkAZNmyYWruudsaMGZoxY0aLfYMGDdKOHTvCfVoAAHAV4bd4AACAdQgoAADAOgQUAABgnYjeSRaIlKvtHioAcLXhDAoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1gk7oGzbtk1jx45Venq6XC6X1qxZE9L/0EMPyeVyhTxGjx4dMubUqVOaPHmyvF6vEhMTNW3aNNXX11/WjgAAgCtH2AGloaFB2dnZWrx48UXHjB49WtXV1cHHb3/725D+yZMn66OPPlJZWZnWrl2rbdu2acaMGeHPHgAAXJFiw12hoKBABQUFlxzj8XiUmpraYt/HH3+sdevW6YMPPtCQIUMkSc8//7zGjBmjX/3qV0pPTw93SgAA4AoTdkBpiy1btiglJUU9evTQd7/7Xf3iF79Qz549JUnl5eVKTEwMhhNJysvLU0xMjCoqKnTvvfc2257f75ff7w8u+3w+SVIgEFAgEAi2X/j7q21on46opaeLidi2L6Wjjw+OS+dQS+dQS+dQy7YLp0aOB5TRo0frvvvuU1ZWlg4ePKif//znKigoUHl5ubp06aKamhqlpKSETiI2VklJSaqpqWlxmyUlJSouLm7WvmHDBsXHxzdrLysrc2ZnENFaPj00Ypu+pLfeeisqz8tx6Rxq6Rxq6Rxq2brGxsY2j3U8oEyYMCH492233aYBAwbo+uuv15YtWzRixIh2bXPevHkqLCwMLvt8PmVkZGjkyJHyer3B9kAgoLKyMuXn58vtdrd/J9AhtexftD4i223NvqJRHfp8HJfOoZbOoZbOoZZtd+ETkLaIyEc8X3XdddcpOTlZVVVVGjFihFJTU3X8+PGQMWfPntWpU6cuet2Kx+ORx+Np1u52u1s8GC7WjvBFspb+c66IbLc1N87f0O51Dy+8u93rclw6h1o6h1o6h1q2Lpz6RPw+KH/605908uRJpaWlSZJyc3NVW1urXbt2Bcds3rxZTU1NysnJifR0AABAJxD2GZT6+npVVVUFlw8dOqTKykolJSUpKSlJxcXFGj9+vFJTU3Xw4EH97Gc/0w033KBRo86fVr/llls0evRoTZ8+XUuXLlUgENDs2bM1YcIEvsEDAAAkteMMys6dOzVw4EANHDhQklRYWKiBAwfqiSeeUJcuXbR3717dc889uummmzRt2jQNHjxY7777bshHNCtXrlS/fv00YsQIjRkzRnfeeaeWLVvm3F4BAIBOLewzKMOGDZMxF/966Pr1rV/4mJSUpNLS0nCfGgAAXCX4LR4AAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOmEHlG3btmns2LFKT0+Xy+XSmjVrgn2BQEBz587Vbbfdpm984xtKT0/X3/3d3+no0aMh2+jbt69cLlfIY+HChZe9MwAA4MoQdkBpaGhQdna2Fi9e3KyvsbFRu3fv1vz587V79269+uqr2r9/v+65555mYxcsWKDq6urg45FHHmnfHgAAgCtObLgrFBQUqKCgoMW+7t27q6ysLKTtX//1XzV06FAdOXJEmZmZwfaEhASlpqa26Tn9fr/8fn9w2efzSTp/xiYQCATbL/z91Ta0T0fU0tPFRGzbkdKeenBcOodaOodaOodatl04NXIZY9r9LuFyubR69WqNGzfuomM2btyokSNHqra2Vl6vV9L5j3jOnDmjQCCgzMxMTZo0SXPmzFFsbMt5qaioSMXFxc3aS0tLFR8f397pAwCADtTY2KhJkyaprq4umAkuJqIB5cyZM7rjjjvUr18/rVy5Mti+aNEiDRo0SElJSXrvvfc0b948TZ06VYsWLWpxOy2dQcnIyNCJEydCdjAQCKisrEz5+flyu93t3S2oY2rZv2h9RLYbSfuKRoW9Dselc6ilc6ilc6hl2/l8PiUnJ7cpoIT9EU9bBQIB/e3f/q2MMVqyZElIX2FhYfDvAQMGKC4uTj/84Q9VUlIij8fTbFsej6fFdrfb3eLBcLF2hK+1WvZ97M3L2LrrMtaNjss5rjgunUMtnUMtnUMtWxdOfSLyNeML4eTTTz9VWVlZqykpJydHZ8+e1eHDhyMxHQAA0Mk4fgblQjg5cOCA3nnnHfXs2bPVdSorKxUTE6OUlBSnpwMAADqhsANKfX29qqqqgsuHDh1SZWWlkpKSlJaWpu9///vavXu31q5dq3PnzqmmpkaSlJSUpLi4OJWXl6uiokLDhw9XQkKCysvLNWfOHD344IPq0aOHc3sGAAA6rbADys6dOzV8+PDg8oXrSaZMmaKioiK9/vrrkqTbb789ZL133nlHw4YNk8fj0apVq1RUVCS/36+srCzNmTMn5LoUAABwdQs7oAwbNkyX+uJPa18KGjRokHbs2BHu0wIAgKsIv8UDAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsE7YAWXbtm0aO3as0tPT5XK5tGbNmpB+Y4yeeOIJpaWlqVu3bsrLy9OBAwdCxpw6dUqTJ0+W1+tVYmKipk2bpvr6+svaEQAAcOUIO6A0NDQoOztbixcvbrH/6aef1nPPPaelS5eqoqJC3/jGNzRq1CidOXMmOGby5Mn66KOPVFZWprVr12rbtm2aMWNG+/cCAABcUWLDXaGgoEAFBQUt9hlj9Oyzz+rxxx/X9773PUnSf/7nf6p3795as2aNJkyYoI8//ljr1q3TBx98oCFDhkiSnn/+eY0ZM0a/+tWvlJ6efhm7AwAArgRhB5RLOXTokGpqapSXlxds6969u3JyclReXq4JEyaovLxciYmJwXAiSXl5eYqJiVFFRYXuvffeZtv1+/3y+/3BZZ/PJ0kKBAIKBALB9gt/f7UN7dPWWnq6mI6YjjXac2xxXDqHWjqHWjqHWrZdODVyNKDU1NRIknr37h3S3rt372BfTU2NUlJSQicRG6ukpKTgmK8rKSlRcXFxs/YNGzYoPj6+WXtZWVm75o/mWqvl00M7aCKWeOutt9q9Lselc6ilc6ilc6hl6xobG9s81tGAEinz5s1TYWFhcNnn8ykjI0MjR46U1+sNtgcCAZWVlSk/P19utzsaU71itLWW/YvWd+Csom9f0aiw1+G4dA61dA61dA61bLsLn4C0haMBJTU1VZJ07NgxpaWlBduPHTum22+/PTjm+PHjIeudPXtWp06dCq7/dR6PRx6Pp1m72+1u8WC4WDvC11ot/edcHTib6Luc44rj0jnU0jnU0jnUsnXh1MfR+6BkZWUpNTVVmzZtCrb5fD5VVFQoNzdXkpSbm6va2lrt2rUrOGbz5s1qampSTk6Ok9MBAACdVNhnUOrr61VVVRVcPnTokCorK5WUlKTMzEw9+uij+sUvfqEbb7xRWVlZmj9/vtLT0zVu3DhJ0i233KLRo0dr+vTpWrp0qQKBgGbPnq0JEybwDR4AACCpHQFl586dGj58eHD5wrUhU6ZM0UsvvaSf/exnamho0IwZM1RbW6s777xT69atU9euXYPrrFy5UrNnz9aIESMUExOj8ePH67nnnnNgdwAAwJUg7IAybNgwGXPxr5a6XC4tWLBACxYsuOiYpKQklZaWhvvUAADgKsFv8QAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOmH/Fg9wNev72Jthr+PpYvT00AhMBgCuYJxBAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOo4HlL59+8rlcjV7zJo1S5I0bNiwZn0zZ850ehoAAKATi3V6gx988IHOnTsXXN63b5/y8/N1//33B9umT5+uBQsWBJfj4+OdngYAAOjEHA8ovXr1ClleuHChrr/+en3nO98JtsXHxys1NdXppwYAAFcIxwPKV3355ZdasWKFCgsL5XK5gu0rV67UihUrlJqaqrFjx2r+/PmXPIvi9/vl9/uDyz6fT5IUCAQUCASC7Rf+/mob2qettfR0MR0xnU7NE3O+RhyXl4/XuHOopXOoZduFUyOXMSZi7zC///3vNWnSJB05ckTp6emSpGXLlqlPnz5KT0/X3r17NXfuXA0dOlSvvvrqRbdTVFSk4uLiZu2lpaV8PAQAQCfR2NioSZMmqa6uTl6v95JjIxpQRo0apbi4OL3xxhsXHbN582aNGDFCVVVVuv7661sc09IZlIyMDJ04cSJkBwOBgMrKypSfny+32+3cjlyF2lrL/kXrO3BWnZMnxuipIU0clw7gNe4caukcatl2Pp9PycnJbQooEfuI59NPP9XGjRsveWZEknJyciTpkgHF4/HI4/E0a3e73S0eDBdrR/haq6X/nOuifQjFcekcaukcaukcatm6cOoTsfugLF++XCkpKbr77rsvOa6yslKSlJaWFqmpAACATiYiZ1Campq0fPlyTZkyRbGxf3mKgwcPqrS0VGPGjFHPnj21d+9ezZkzR3fddZcGDBgQiakAAIBOKCIBZePGjTpy5Ih+8IMfhLTHxcVp48aNevbZZ9XQ0KCMjAyNHz9ejz/+eCSmAQAAOqmIBJSRI0eqpWtvMzIytHXr1kg8JQAAuILwWzwAAMA6BBQAAGAdAgoAALBORG91j86h72NvNmvzdDF6euj5G7FxrxMAQEfjDAoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArON4QCkqKpLL5Qp59OvXL9h/5swZzZo1Sz179tQ111yj8ePH69ixY05PAwAAdGIROYPyzW9+U9XV1cHH9u3bg31z5szRG2+8oVdeeUVbt27V0aNHdd9990ViGgAAoJOKjchGY2OVmprarL2urk4vvPCCSktL9d3vfleStHz5ct1yyy3asWOHvv3tb0diOgAAoJOJSEA5cOCA0tPT1bVrV+Xm5qqkpESZmZnatWuXAoGA8vLygmP79eunzMxMlZeXXzSg+P1++f3+4LLP55MkBQIBBQKBYPuFv7/ahtZ5upjmbTEm5F+034UaclxePl7jzqGWzqGWbRdOjVzGGEffgd5++23V19fr5ptvVnV1tYqLi/X5559r3759euONNzR16tSQsCFJQ4cO1fDhw/XP//zPLW6zqKhIxcXFzdpLS0sVHx/v5PQBAECENDY2atKkSaqrq5PX673kWMcDytfV1taqT58+WrRokbp169augNLSGZSMjAydOHEiZAcDgYDKysqUn58vt9sdmR26AvUvWt+szRNj9NSQJs3fGSN/kysKs7pyXKglx+Xl4zXuHGrpHGrZdj6fT8nJyW0KKBH5iOerEhMTddNNN6mqqkr5+fn68ssvVVtbq8TExOCYY8eOtXjNygUej0cej6dZu9vtbvFguFg7WuY/d/EA4m9yXbIfbcdx6Rxq6Rxq6Rxq2bpw6hPx+6DU19fr4MGDSktL0+DBg+V2u7Vp06Zg//79+3XkyBHl5uZGeioAAKCTcPwMyk9/+lONHTtWffr00dGjR/Xkk0+qS5cumjhxorp3765p06apsLBQSUlJ8nq9euSRR5Sbm8s3eAAAQJDjAeVPf/qTJk6cqJMnT6pXr1668847tWPHDvXq1UuS9C//8i+KiYnR+PHj5ff7NWrUKP3bv/2b09MAAACdmOMBZdWqVZfs79q1qxYvXqzFixc7/dQAAOAKwW/xAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANZxPKCUlJToW9/6lhISEpSSkqJx48Zp//79IWOGDRsml8sV8pg5c6bTUwEAAJ2U4wFl69atmjVrlnbs2KGysjIFAgGNHDlSDQ0NIeOmT5+u6urq4OPpp592eioAAKCTinV6g+vWrQtZfumll5SSkqJdu3bprrvuCrbHx8crNTXV6acHrNW/aL3851ztWvfwwrsdng0A2M3xgPJ1dXV1kqSkpKSQ9pUrV2rFihVKTU3V2LFjNX/+fMXHx7e4Db/fL7/fH1z2+XySpEAgoEAgEGy/8PdX29A6TxfTvC3GhPyL9nOilhzT5/Eadw61dA61bLtwauQyxkTsHaipqUn33HOPamtrtX379mD7smXL1KdPH6Wnp2vv3r2aO3euhg4dqldffbXF7RQVFam4uLhZe2lp6UVDDQAAsEtjY6MmTZqkuro6eb3eS46NaEB5+OGH9fbbb2v79u269tprLzpu8+bNGjFihKqqqnT99dc362/pDEpGRoZOnDgRsoOBQEBlZWXKz8+X2+12dmeuYP2L1jdr88QYPTWkSfN3xsjf1L6PJXCeE7XcVzTK4Vl1TrzGnUMtnUMt287n8yk5OblNASViH/HMnj1ba9eu1bZt2y4ZTiQpJydHki4aUDwejzweT7N2t9vd4sFwsXa07FLXRfibXO2+bgKhLqeWHM+heI07h1o6h1q2Lpz6OB5QjDF65JFHtHr1am3ZskVZWVmtrlNZWSlJSktLc3o6AACgE3I8oMyaNUulpaV67bXXlJCQoJqaGklS9+7d1a1bNx08eFClpaUaM2aMevbsqb1792rOnDm66667NGDAAKenc9Xo+9ib0Z4CAACOcTygLFmyRNL5m7F91fLly/XQQw8pLi5OGzdu1LPPPquGhgZlZGRo/Pjxevzxx52eCgAA6KQi8hHPpWRkZGjr1q1OPy0AALiC8Fs8AADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDqx0Z4A/qLvY29GewoAAFiBMygAAMA6BBQAAGAdAgoAALAO16A4jOtIAAC4fJxBAQAA1uEMCtAJXM6ZucML73ZwJgDQMTiDAgAArENAAQAA1iGgAAAA63ANCnCF4/oVAJ0RZ1AAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANbhPigt4BeJAQCILs6gAAAA6xBQAACAdQgoAADAOlG9BmXx4sV65plnVFNTo+zsbD3//PMaOnRoNKcEoJPjt4eAv+jMr4eonUH53e9+p8LCQj355JPavXu3srOzNWrUKB0/fjxaUwIAAJaIWkBZtGiRpk+frqlTp+rWW2/V0qVLFR8frxdffDFaUwIAAJaIykc8X375pXbt2qV58+YF22JiYpSXl6fy8vJm4/1+v/x+f3C5rq5OknTq1CkFAoFgeyAQUGNjo06ePCm3293u+cWebWj3uleK2CajxsYmxQZidK7JFe3pdGqduZYnT56M9hRCtOU1fjmvX9v2N5Kc+u8l7K6lba+H06dPS5KMMa2OjUpAOXHihM6dO6fevXuHtPfu3Vv/+7//22x8SUmJiouLm7VnZWVFbI6QJkV7AleQzlrL5F9HewYd62rbX+BSIvl6OH36tLp3737JMZ3iRm3z5s1TYWFhcLmpqUmnTp1Sz5495XL95f9IfT6fMjIy9Nlnn8nr9UZjqlcMaukcaukcaukcaukcatl2xhidPn1a6enprY6NSkBJTk5Wly5ddOzYsZD2Y8eOKTU1tdl4j8cjj8cT0paYmHjR7Xu9Xg4Sh1BL51BL51BL51BL51DLtmntzMkFUblINi4uToMHD9amTZuCbU1NTdq0aZNyc3OjMSUAAGCRqH3EU1hYqClTpmjIkCEaOnSonn32WTU0NGjq1KnRmhIAALBE1ALKAw88oD//+c964oknVFNTo9tvv13r1q1rduFsODwej5588slmHwchfNTSOdTSOdTSOdTSOdQyMlymLd/1AQAA6ED8Fg8AALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOtYH1C2bdumsWPHKj09XS6XS2vWrAnpN8boiSeeUFpamrp166a8vDwdOHAgZMypU6c0efJkeb1eJSYmatq0aaqvr+/AvbBDSUmJvvWtbykhIUEpKSkaN26c9u/fHzLmzJkzmjVrlnr27KlrrrlG48ePb3bH3yNHjujuu+9WfHy8UlJS9A//8A86e/ZsR+5K1C1ZskQDBgwI3jkyNzdXb7/9drCfOrbPwoUL5XK59OijjwbbqGXbFRUVyeVyhTz69esX7KeWbff555/rwQcfVM+ePdWtWzfddttt2rlzZ7Cf954OYCz31ltvmX/8x380r776qpFkVq9eHdK/cOFC0717d7NmzRrz3//93+aee+4xWVlZ5osvvgiOGT16tMnOzjY7duww7777rrnhhhvMxIkTO3hPom/UqFFm+fLlZt++faaystKMGTPGZGZmmvr6+uCYmTNnmoyMDLNp0yazc+dO8+1vf9v89V//dbD/7Nmzpn///iYvL8/s2bPHvPXWWyY5OdnMmzcvGrsUNa+//rp58803zSeffGL2799vfv7znxu322327dtnjKGO7fH++++bvn37mgEDBpgf//jHwXZq2XZPPvmk+eY3v2mqq6uDjz//+c/BfmrZNqdOnTJ9+vQxDz30kKmoqDB//OMfzfr1601VVVVwDO89kWd9QPmqrweUpqYmk5qaap555plgW21trfF4POa3v/2tMcaY//mf/zGSzAcffBAc8/bbbxuXy2U+//zzDpu7jY4fP24kma1btxpjztfO7XabV155JTjm448/NpJMeXm5MeZ8YIyJiTE1NTXBMUuWLDFer9f4/f6O3QHL9OjRw/zHf/wHdWyH06dPmxtvvNGUlZWZ73znO8GAQi3D8+STT5rs7OwW+6hl282dO9fceeedF+3nvadjWP8Rz6UcOnRINTU1ysvLC7Z1795dOTk5Ki8vlySVl5crMTFRQ4YMCY7Jy8tTTEyMKioqOnzONqmrq5MkJSUlSZJ27dqlQCAQUs9+/fopMzMzpJ633XZbyB1/R40aJZ/Pp48++qgDZ2+Pc+fOadWqVWpoaFBubi51bIdZs2bp7rvvDqmZxDHZHgcOHFB6erquu+46TZ48WUeOHJFELcPx+uuva8iQIbr//vuVkpKigQMH6t///d+D/bz3dIxOHVBqamokqdnt8Xv37h3sq6mpUUpKSkh/bGyskpKSgmOuRk1NTXr00Ud1xx13qH///pLO1youLq7ZL0V/vZ4t1ftC39Xkww8/1DXXXCOPx6OZM2dq9erVuvXWW6ljmFatWqXdu3erpKSkWR+1DE9OTo5eeuklrVu3TkuWLNGhQ4f0N3/zNzp9+jS1DMMf//hHLVmyRDfeeKPWr1+vhx9+WD/60Y/08ssvS+K9p6NE7bd4EF2zZs3Svn37tH379mhPpdO6+eabVVlZqbq6Ov3hD3/QlClTtHXr1mhPq1P57LPP9OMf/1hlZWXq2rVrtKfT6RUUFAT/HjBggHJyctSnTx/9/ve/V7du3aI4s86lqalJQ4YM0T/90z9JkgYOHKh9+/Zp6dKlmjJlSpRnd/Xo1GdQUlNTJanZVejHjh0L9qWmpur48eMh/WfPntWpU6eCY642s2fP1tq1a/XOO+/o2muvDbanpqbqyy+/VG1tbcj4r9ezpXpf6LuaxMXF6YYbbtDgwYNVUlKi7Oxs/eY3v6GOYdi1a5eOHz+uQYMGKTY2VrGxsdq6dauee+45xcbGqnfv3tTyMiQmJuqmm25SVVUVx2UY0tLSdOutt4a03XLLLcGPy3jv6RidOqBkZWUpNTVVmzZtCrb5fD5VVFQoNzdXkpSbm6va2lrt2rUrOGbz5s1qampSTk5Oh885mowxmj17tlavXq3NmzcrKysrpH/w4MFyu90h9dy/f7+OHDkSUs8PP/ww5IVXVlYmr9fb7AV9tWlqapLf76eOYRgxYoQ+/PBDVVZWBh9DhgzR5MmTg39Ty/arr6/XwYMHlZaWxnEZhjvuuKPZLRg++eQT9enTRxLvPR0m2lfptub06dNmz549Zs+ePUaSWbRokdmzZ4/59NNPjTHnv+qVmJhoXnvtNbN3717zve99r8Wveg0cONBUVFSY7du3mxtvvPGq/KrXww8/bLp37262bNkS8jXExsbG4JiZM2eazMxMs3nzZrNz506Tm5trcnNzg/0XvoY4cuRIU1lZadatW2d69ep11X0N8bHHHjNbt241hw4dMnv37jWPPfaYcblcZsOGDcYY6ng5vvotHmOoZTh+8pOfmC1btphDhw6Z//qv/zJ5eXkmOTnZHD9+3BhDLdvq/fffN7GxseaXv/ylOXDggFm5cqWJj483K1asCI7hvSfyrA8o77zzjpHU7DFlyhRjzPmve82fP9/07t3beDweM2LECLN///6QbZw8edJMnDjRXHPNNcbr9ZqpU6ea06dPR2FvoqulOkoyy5cvD4754osvzN///d+bHj16mPj4eHPvvfea6urqkO0cPnzYFBQUmG7dupnk5GTzk5/8xAQCgQ7em+j6wQ9+YPr06WPi4uJMr169zIgRI4LhxBjqeDm+HlCoZds98MADJi0tzcTFxZm/+qu/Mg888EDIvTuoZdu98cYbpn///sbj8Zh+/fqZZcuWhfTz3hN5LmOMic65GwAAgJZ16mtQAADAlYmAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADW+X93U5a4KAP3LQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "LEN = []\n",
+    "for i in range(877):\n",
+    "    LEN.append(len(dataset_text['text'][i]))\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "pd.DataFrame(np.array(LEN)).hist(bins = 30)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tokenization of the dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b9919f91c04f49428be62ac33921ec7d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/877 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['input_ids', 'attention_mask'],\n",
+       "    num_rows: 877\n",
+       "})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.1\")\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "tokenizer.padding_side = \"right\" \n",
+    "\n",
+    "def tokenize_function(examples):\n",
+    "    result = tokenizer(examples[\"text\"])\n",
+    "    return result\n",
+    "\n",
+    "tokenized_datasets = dataset_text.map(\n",
+    "    tokenize_function, batched=True, remove_columns=[\"text\"]\n",
+    ")\n",
+    "tokenized_datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "tokenizer.mask_token = '<MASK>'\n",
+    "collator = DataCollatorForLanguageModeling(mlm = True,mlm_probability=0.15,tokenizer = tokenizer)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "rjOMoSbGSxx9"
+   },
+   "source": [
+    "# Foundation model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "id": "ZwXZbQ2dSwzI",
+    "outputId": "a57e521a-a8a3-48e9-a478-63334083f94a"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "87d196af4c864c2f9381a18ceb5720e5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "bnb_config = BitsAndBytesConfig(\n",
+    "    load_in_4bit=True,\n",
+    "    bnb_4bit_quant_type=\"nf4\",\n",
+    "    bnb_4bit_compute_dtype=torch.float16,\n",
+    ")\n",
+    "\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "        \"mistralai/Mistral-7B-Instruct-v0.1\",\n",
+    "        device_map=\"auto\",\n",
+    "        torch_dtype=torch.float16, #torch.bfloat16,\n",
+    "        trust_remote_code=True\n",
+    "            )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "NuAx3zBeUL1q"
+   },
+   "source": [
+    "## LoRa configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "id": "dQdvjTYTT1vQ",
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "lora_alpha = 16\n",
+    "lora_dropout = 0.1\n",
+    "lora_r = 64\n",
+    "\n",
+    "peft_config = LoraConfig(\n",
+    "    lora_alpha=lora_alpha,\n",
+    "    lora_dropout=lora_dropout,\n",
+    "    r=lora_r,\n",
+    "    bias=\"none\",\n",
+    "    task_type=\"CAUSAL_LM\",\n",
+    "    target_modules=[\n",
+    "        \"Wqkv\",\n",
+    "        \"out_proj\",\n",
+    "        \"up_proj\",\n",
+    "        \"down_proj\",\n",
+    "    ])\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Training parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "output_dir = \"/MY_DIRECTORY\"\n",
+    "per_device_train_batch_size = 1\n",
+    "gradient_accumulation_steps = 16 \n",
+    "optim = \"paged_adamw_32bit\"\n",
+    "save_steps = 55 \n",
+    "logging_steps = 55\n",
+    "learning_rate = 1e-4\n",
+    "max_grad_norm = 0.3\n",
+    "max_steps = 55 * 15 \n",
+    "warmup_ratio = 0.03\n",
+    "lr_scheduler_type = \"linear\"\n",
+    "\n",
+    "training_arguments = TrainingArguments(\n",
+    "    output_dir=output_dir,\n",
+    "    per_device_train_batch_size=per_device_train_batch_size,\n",
+    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
+    "    optim=optim,\n",
+    "    logging_steps=logging_steps,\n",
+    "    save_strategy= 'no', #''epoch',\n",
+    "    #save_steps=save_steps,\n",
+    "    #evaluation_strategy = \"steps\",#\"epoch\",\n",
+    "    learning_rate=learning_rate,\n",
+    "    fp16=True,\n",
+    "    max_grad_norm=max_grad_norm,\n",
+    "    max_steps=max_steps,\n",
+    "    warmup_ratio=warmup_ratio,\n",
+    "    group_by_length=True,\n",
+    "    lr_scheduler_type=lr_scheduler_type,\n",
+    "    report_to = 'none',\n",
+    "    save_total_limit = 1\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = get_peft_model(model, peft_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    tokenizer=tokenizer,\n",
+    "    data_collator=collator,\n",
+    "    train_dataset=tokenized_datasets,\n",
+    "    #eval_dataset=\n",
+    "    args=training_arguments,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Save model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-11-12T18:43:12.964677Z",
+     "iopub.status.busy": "2023-11-12T18:43:12.964270Z",
+     "iopub.status.idle": "2023-11-12T18:43:13.685390Z",
+     "shell.execute_reply": "2023-11-12T18:43:13.684268Z",
+     "shell.execute_reply.started": "2023-11-12T18:43:12.964645Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "trainer.save_model(output_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Reload the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model1 = AutoPeftModelForCausalLM.from_pretrained(output_dir, load_in_4bit=True)\n",
+    "tokenizer1 = AutoTokenizer.from_pretrained(output_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Prompt preparation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Criteria for early stopping during generation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import StoppingCriteria,StoppingCriteriaList"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class StopOnTokens(StoppingCriteria):\n",
+    "    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n",
+    "        stop_ids = [28723] # corresponding to '.'\n",
+    "        for stop_id in stop_ids:\n",
+    "            if input_ids[0][-1] == stop_id:\n",
+    "                return True\n",
+    "        return False"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Prompt answers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "text = \"At Laurent restaurant : do you have any vegetarian options?\"\n",
+    "#text = \"At Laurent restaurant: do you have Apple pie?\"\n",
+    "#text = \"At Laurent restaurant: what is included in the Premium Sweetheart Set for Two?\"\n",
+    "#text = \"At Laurent restaurant: do you have Seafood Paella?\"\n",
+    "#text = \"At Laurent restaurant: what is the best menu?\"\n",
+    "\n",
+    "inputs = tokenizer1(text, return_tensors=\"pt\").to('cuda')\n",
+    "out = model1.generate(**inputs, \n",
+    "                     pad_token_id=tokenizer.eos_token_id,\n",
+    "                     stopping_criteria = StoppingCriteriaList([StopOnTokens()]),\n",
+    "                     max_new_tokens=100\n",
+    "                    )\n",
+    "\n",
+    "tokenizer1.decode(out[0],skip_special_tokens=True).split(\"[answer]:\")[1]\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}