Upload 1_Bit_LLM_Pretraining.ipynb

1_Bit_LLM_Pretraining.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 90,
+   "execution_count": 1,
    "metadata": {
     "id": "dbsnrDKKVarI",
     "colab": {
@@ -123,8 +123,8 @@
     },
     "outputId": "5808189b-e624-42d7-856f-bc3b0201fab9",
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:59:34.679348Z",
-     "start_time": "2024-04-16T22:59:19.314163Z"
+     "end_time": "2024-04-16T23:12:05.968918Z",
+     "start_time": "2024-04-16T23:11:31.417421Z"
     }
    },
    "outputs": [
@@ -133,23 +133,9 @@
      "output_type": "stream",
      "text": [
       "Requirement already satisfied: datasets in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (2.18.0)\n",
-      "Requirement already satisfied: wandb in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (0.16.6)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "wandb: WARNING Calling wandb.login() after wandb.init() has no effect.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "Requirement already satisfied: wandb in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (0.16.6)\n",
       "Requirement already satisfied: accelerate in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (0.28.0)\n",
-      "Token has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.Requirement already satisfied: filelock in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from datasets) (3.9.0)\n",
-      "\n",
+      "Requirement already satisfied: filelock in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from datasets) (3.9.0)\n",
       "Requirement already satisfied: numpy>=1.17 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from datasets) (1.23.5)\n",
       "Requirement already satisfied: pyarrow>=12.0.0 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from datasets) (15.0.2)\n",
       "Requirement already satisfied: pyarrow-hotfix in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from datasets) (0.6)\n",
@@ -197,7 +183,24 @@
       "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from pandas->datasets) (2024.1)\n",
       "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.1)\n",
       "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
-      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
+      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\saad.naeem\\appdata\\local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "wandb: Currently logged in as: saadnaeem-dev. Use `wandb login --relogin` to force relogin\n",
+      "wandb: WARNING If you're specifying your api key in code, ensure this code is not shared publicly.\n",
+      "wandb: WARNING Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
+      "wandb: Appending key for api.wandb.ai to your netrc file: C:\\Users\\saad.naeem\\.netrc\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Token has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
       "Token is valid (permission: write).\n",
       "Your token has been saved to C:\\Users\\saad.naeem\\.cache\\huggingface\\token\n",
       "Login successful\n"
@@ -319,11 +322,11 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:55:40.522736Z",
-     "start_time": "2024-04-16T22:55:39.803487Z"
+     "end_time": "2024-04-16T23:12:06.491480Z",
+     "start_time": "2024-04-16T23:12:05.970774Z"
     }
    },
-   "execution_count": 80
+   "execution_count": 2
   },
   {
    "cell_type": "code",
@@ -332,7 +335,7 @@
      "data": {
       "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['input_ids'],\n        num_rows: 476702\n    })\n})"
      },
-     "execution_count": 81,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -343,11 +346,11 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:55:41.976375Z",
-     "start_time": "2024-04-16T22:55:41.955375Z"
+     "end_time": "2024-04-16T23:12:06.507322Z",
+     "start_time": "2024-04-16T23:12:06.492328Z"
     }
    },
-   "execution_count": 81
+   "execution_count": 3
   },
   {
    "cell_type": "code",
@@ -361,17 +364,17 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:55:43.097378Z",
-     "start_time": "2024-04-16T22:55:43.076377Z"
+     "end_time": "2024-04-16T23:12:06.523338Z",
+     "start_time": "2024-04-16T23:12:06.509324Z"
     }
    },
-   "execution_count": 82
+   "execution_count": 4
   },
   {
    "cell_type": "code",
    "outputs": [],
    "source": [
-    "sampled_dataset = tokenized_data['train'].select(range(1000))\n",
+    "sampled_dataset = tokenized_data['train'].select(range(500))\n",
     "sampled_dataset_dict = DatasetDict({\n",
     "    'train': sampled_dataset\n",
     "})"
@@ -379,11 +382,11 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:55:44.559805Z",
-     "start_time": "2024-04-16T22:55:44.537823Z"
+     "end_time": "2024-04-16T23:12:06.539322Z",
+     "start_time": "2024-04-16T23:12:06.525349Z"
     }
    },
-   "execution_count": 83
+   "execution_count": 5
   },
   {
    "cell_type": "code",
@@ -393,7 +396,7 @@
       "text/plain": "                                           input_ids\n0  [1, 2266, 338, 385, 6597, 515, 263, 24499, 299...",
       "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>input_ids</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>[1, 2266, 338, 385, 6597, 515, 263, 24499, 299...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 85,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -404,20 +407,20 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:56:05.152275Z",
-     "start_time": "2024-04-16T22:56:05.132254Z"
+     "end_time": "2024-04-16T23:12:06.570322Z",
+     "start_time": "2024-04-16T23:12:06.540322Z"
     }
    },
-   "execution_count": 85
+   "execution_count": 6
   },
   {
    "cell_type": "code",
    "outputs": [
     {
      "data": {
-      "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['input_ids'],\n        num_rows: 1000\n    })\n})"
+      "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['input_ids'],\n        num_rows: 500\n    })\n})"
      },
-     "execution_count": 86,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -429,11 +432,11 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:56:06.004257Z",
-     "start_time": "2024-04-16T22:56:05.990254Z"
+     "end_time": "2024-04-16T23:12:06.586322Z",
+     "start_time": "2024-04-16T23:12:06.572323Z"
     }
    },
-   "execution_count": 86
+   "execution_count": 7
   },
   {
    "cell_type": "code",
@@ -442,7 +445,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Training on 256_000 tokens\n",
+      "Training on 128_000 tokens\n",
       "Model size: 77.5M parameters\n"
      ]
     },
@@ -511,28 +514,80 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:56:11.670768Z",
-     "start_time": "2024-04-16T22:56:09.804760Z"
+     "end_time": "2024-04-16T23:12:10.698564Z",
+     "start_time": "2024-04-16T23:12:09.118565Z"
     }
    },
-   "execution_count": 87
+   "execution_count": 8
   },
   {
    "cell_type": "code",
    "outputs": [
+    {
+     "data": {
+      "text/plain": "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011111111111111112, max=1.0…",
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "a2723ee61bae4cdaaaed5fc2553ead54"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": "<IPython.core.display.HTML object>",
+      "text/html": "Tracking run with wandb version 0.16.6"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": "<IPython.core.display.HTML object>",
+      "text/html": "Run data is saved locally in <code>C:\\Users\\saad.naeem\\PycharmProjects\\NLP-Projects-NHV-1-Bit-LLM\\NLP-Projects-NHV-main\\LLMs Related\\Era of 1 Bit LLMs\\wandb\\run-20240417_041212-qepzjjtf</code>"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": "<IPython.core.display.HTML object>",
+      "text/html": "Syncing run <strong><a href='https://wandb.ai/saadnaeem-dev/huggingface/runs/qepzjjtf' target=\"_blank\">fancy-disco-3</a></strong> to <a href='https://wandb.ai/saadnaeem-dev/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": "<IPython.core.display.HTML object>",
+      "text/html": " View project at <a href='https://wandb.ai/saadnaeem-dev/huggingface' target=\"_blank\">https://wandb.ai/saadnaeem-dev/huggingface</a>"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": "<IPython.core.display.HTML object>",
+      "text/html": " View run at <a href='https://wandb.ai/saadnaeem-dev/huggingface/runs/qepzjjtf' target=\"_blank\">https://wandb.ai/saadnaeem-dev/huggingface/runs/qepzjjtf</a>"
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "data": {
       "text/plain": "<IPython.core.display.HTML object>",
-      "text/html": "\n    <div>\n      \n      <progress value='16' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [16/16 01:46, Epoch 1/1]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Step</th>\n      <th>Training Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"
+      "text/html": "\n    <div>\n      \n      <progress value='8' max='8' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [8/8 00:38, Epoch 1/1]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Step</th>\n      <th>Training Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "data": {
-      "text/plain": "TrainOutput(global_step=16, training_loss=9.391032218933105, metrics={'train_runtime': 110.6973, 'train_samples_per_second': 9.034, 'train_steps_per_second': 0.145, 'total_flos': 81244717056000.0, 'train_loss': 9.391032218933105, 'epoch': 1.0})"
+      "text/plain": "TrainOutput(global_step=8, training_loss=10.012518882751465, metrics={'train_runtime': 59.7786, 'train_samples_per_second': 8.364, 'train_steps_per_second': 0.134, 'total_flos': 40622358528000.0, 'train_loss': 10.012518882751465, 'epoch': 1.0})"
      },
-     "execution_count": 88,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -543,17 +598,17 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T22:58:19.694121Z",
-     "start_time": "2024-04-16T22:56:28.225808Z"
+     "end_time": "2024-04-16T23:13:12.309648Z",
+     "start_time": "2024-04-16T23:12:12.383042Z"
     }
    },
-   "execution_count": 88
+   "execution_count": 9
   },
   {
    "cell_type": "code",
    "source": [
-    "trainer.save_model(f\"{output_path}\")\n",
-    "folder = f\"{output_path}\"\n",
+    "trainer.save_model(f\"{output_path}/final_model\")\n",
+    "folder = f\"{output_path}/final_model\"\n",
     "api = HfApi()\n",
     "create_repo(\n",
     "    repo_id = f\"{HUGGINGFACE_ID}/{NEW_MODEL}\",\n",
@@ -622,68 +677,20 @@
     },
     "id": "mnHZU06l5tG3",
     "outputId": "bfa63618-ae11-4415-a695-0349dfecf4ad",
-    "is_executing": true,
     "ExecuteTime": {
-     "start_time": "2024-04-16T22:59:36.439911Z"
+     "end_time": "2024-04-16T23:15:09.601323Z",
+     "start_time": "2024-04-16T23:13:25.238137Z"
     }
    },
-   "execution_count": null,
+   "execution_count": 10,
    "outputs": [
-    {
-     "data": {
-      "text/plain": "Upload 9 LFS files:   0%|          | 0/9 [00:00<?, ?it/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "d671df963340448d8c4c0e5c55a09504"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": "optimizer.pt:   0%|          | 0.00/620M [00:00<?, ?B/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "778d8e2e14334fe4ad0bd0346c71a88c"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": "optimizer.pt:   0%|          | 0.00/620M [00:00<?, ?B/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "d8d15dda98a54be6acca1bb882bf125f"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": "model.safetensors:   0%|          | 0.00/310M [00:00<?, ?B/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "8c71d439fb7540f99a19cc05a4aaa3bd"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
     {
      "data": {
       "text/plain": "model.safetensors:   0%|          | 0.00/310M [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "4f4f038a3dee47bc953e8b5d944e8ce2"
+       "model_id": "1b01daad6e944bbaa194f0181b0a2af6"
       }
      },
      "metadata": {},
@@ -691,15 +698,11 @@
     },
     {
      "data": {
-      "text/plain": "model.safetensors:   0%|          | 0.00/310M [00:00<?, ?B/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "981fb82e520e414fac786f4a2c3192f6"
-      }
+      "text/plain": "CommitInfo(commit_url='https://huggingface.co/saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained/commit/13b5f27c104838f8e8c1a1f0221aa1e378eb97fd', commit_message='Upload folder using huggingface_hub', commit_description='', oid='13b5f27c104838f8e8c1a1f0221aa1e378eb97fd', pr_url=None, pr_revision=None, pr_num=None)"
      },
+     "execution_count": 10,
      "metadata": {},
-     "output_type": "display_data"
+     "output_type": "execute_result"
     }
    ]
   },
@@ -770,327 +773,42 @@
      "height": 107
     },
     "id": "wtB3ZOBB_8E6",
-    "outputId": "39e3df74-5ade-4ff1-e997-28042e178dde"
-   },
-   "execution_count": null,
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stderr",
-     "text": [
-      "Some weights of LlamaForCausalLM were not initialized from the model checkpoint at abideen/Bitnet-Llama-70M and are newly initialized: ['model.layers.0.input_layernorm.weight', 'model.layers.1.input_layernorm.weight', 'model.layers.2.input_layernorm.weight', 'model.layers.3.input_layernorm.weight', 'model.layers.4.input_layernorm.weight', 'model.layers.5.input_layernorm.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
-    {
-     "output_type": "execute_result",
-     "data": {
-      "text/plain": [
-       "'What is Machine Learning?\\n\\nIn today’s digital age, machine learning has become a crucial aspect of our lives. With the increasing popularity of machine learning, machine learning has become a powerful tool for learning and learning. With the'"
-      ],
-      "application/vnd.google.colaboratory.intrinsic+json": {
-       "type": "string"
-      }
-     },
-     "metadata": {},
-     "execution_count": 6
-    }
-   ]
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "prompt = \"Write a short poem\"\n",
-    "inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
-    "generate_ids = model.generate(inputs.input_ids, max_length=50)\n",
-    "tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
-   ],
-   "metadata": {
-    "id": "nQya_hPJEa2M",
+    "outputId": "39e3df74-5ade-4ff1-e997-28042e178dde",
     "ExecuteTime": {
-     "end_time": "2024-04-16T19:10:03.797580Z",
-     "start_time": "2024-04-16T19:10:01.243070Z"
+     "end_time": "2024-04-16T23:19:24.104593Z",
+     "start_time": "2024-04-16T23:18:02.342539Z"
     }
    },
-   "execution_count": 4,
+   "execution_count": 11,
    "outputs": [
     {
      "data": {
-      "text/plain": "'Write a short poem about a \"The Witcher\" by the author of the book \"The Witcher of the Book of the Book of the Book of the Book of the Book of the Book of the Book of the Book of the'"
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ]
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "LlamaForCausalLM(\n",
-      "  (model): LlamaModel(\n",
-      "    (embed_tokens): Embedding(32001, 768, padding_idx=0)\n",
-      "    (layers): ModuleList(\n",
-      "      (0-5): 6 x LlamaDecoderLayer(\n",
-      "        (self_attn): LlamaAttention(\n",
-      "          (q_proj): Linear(in_features=768, out_features=768, bias=False)\n",
-      "          (k_proj): Linear(in_features=768, out_features=768, bias=False)\n",
-      "          (v_proj): Linear(in_features=768, out_features=768, bias=False)\n",
-      "          (o_proj): Linear(in_features=768, out_features=768, bias=False)\n",
-      "          (rotary_emb): LlamaRotaryEmbedding()\n",
-      "        )\n",
-      "        (mlp): LlamaMLP(\n",
-      "          (gate_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n",
-      "          (up_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n",
-      "          (down_proj): BitLinear(in_features=1024, out_features=768, bias=False)\n",
-      "          (act_fn): SiLU()\n",
-      "        )\n",
-      "        (input_layernorm): Identity()\n",
-      "        (post_attention_layernorm): LlamaRMSNorm()\n",
-      "      )\n",
-      "    )\n",
-      "    (norm): LlamaRMSNorm()\n",
-      "  )\n",
-      "  (lm_head): Linear(in_features=768, out_features=32001, bias=False)\n",
-      ")\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(model)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:10:30.619003Z",
-     "start_time": "2024-04-16T19:10:30.598004Z"
-    }
-   },
-   "execution_count": 5
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Model size: 77.5M parameters\n"
-     ]
-    }
-   ],
-   "source": [
-    "# print number of parameters\n",
-    "model_size = sum(t.numel() for t in model.parameters())\n",
-    "print(f\"Model size: {model_size/1000**2:.1f}M parameters\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:11:02.552839Z",
-     "start_time": "2024-04-16T19:11:02.539841Z"
-    }
-   },
-   "execution_count": 6
-  },
-  {
-   "cell_type": "code",
-   "outputs": [],
-   "source": [
-    "# Save the model to disk\n",
-    "import torch\n",
-    "\n",
-    "# Assuming that `model` is your model\n",
-    "torch.save(model.state_dict(), 'Llama2-70M-Cosmopedia-100k-Pretrain.pth')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:31:47.960780Z",
-     "start_time": "2024-04-16T19:31:47.016780Z"
-    }
-   },
-   "execution_count": 8
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "'pwd' is not recognized as an internal or external command,\n",
-      "operable program or batch file.\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pwd"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:32:17.975619Z",
-     "start_time": "2024-04-16T19:32:17.707622Z"
-    }
-   },
-   "execution_count": 9
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "transformers.trainer.Trainer"
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "Trainer"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:49:52.040088Z",
-     "start_time": "2024-04-16T19:49:52.027087Z"
-    }
-   },
-   "execution_count": 12
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "(('Llama2-70M-Cosmopedia-100k-Pretrained\\\\tokenizer_config.json',\n  'Llama2-70M-Cosmopedia-100k-Pretrained\\\\special_tokens_map.json',\n  'Llama2-70M-Cosmopedia-100k-Pretrained\\\\tokenizer.json'),)"
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainer.save_model(\"Llama2-70M-Cosmopedia-100k-Pretrained\")\n",
-    "tokenizer.save_pretrained(\"Llama2-70M-Cosmopedia-100k-Pretrained\"),"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T19:52:28.336234Z",
-     "start_time": "2024-04-16T19:52:14.732448Z"
-    }
-   },
-   "execution_count": 14
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Testing model from Huggingface Hub"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "\"what is machine learning' di'''.icaiaian, isé isé\\ninestinieninamentWriteinieningienAienAest\\ninamenterninest\\ninest\\ninest\\ninament\""
-     },
-     "execution_count": 29,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "prompt = \"what is machine learning\"\n",
-    "inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
-    "generate_ids = model.generate(inputs.input_ids, max_length=50)\n",
-    "tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T20:35:45.132443Z",
-     "start_time": "2024-04-16T20:35:43.770680Z"
-    }
-   },
-   "execution_count": 29
-  },
-  {
-   "cell_type": "code",
-   "outputs": [],
-   "source": [
-    "folder = r\"C:\\Users\\saad.naeem\\PycharmProjects\\NLP-Projects-NHV-1-Bit-LLM\\Llama2-70M-Cosmopedia-100k-Pretrain\"\n",
-    "api = HfApi()\n"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T21:15:39.350857Z",
-     "start_time": "2024-04-16T21:15:39.327854Z"
-    }
-   },
-   "execution_count": 30
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "RepoUrl('https://huggingface.co/saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained', endpoint='https://huggingface.co', repo_type='model', repo_id='saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained')"
-     },
-     "execution_count": 31,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "api.create_repo(\n",
-    "    repo_id = f\"saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained\",\n",
-    "    repo_type=\"model\",\n",
-    "    exist_ok=True\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T21:15:51.034996Z",
-     "start_time": "2024-04-16T21:15:48.858598Z"
-    }
-   },
-   "execution_count": 31
-  },
-  {
-   "cell_type": "code",
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "optimizer.pt:   0%|          | 0.00/620M [00:00<?, ?B/s]",
+      "text/plain": "tokenizer_config.json:   0%|          | 0.00/1.06k [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "217543fd247f45d6b07f99164c0373cb"
+       "model_id": "6a3b01a46be54754acd42a8976327bbf"
       }
      },
      "metadata": {},
      "output_type": "display_data"
     },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\saad.naeem\\AppData\\Local\\anaconda3\\envs\\minerva-prototype\\lib\\site-packages\\huggingface_hub\\file_download.py:149: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\saad.naeem\\.cache\\huggingface\\hub\\models--saadnaeem--Llama2-70M-Cosmopedia-100k-Pretrained. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
+      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
+      "  warnings.warn(message)\n"
+     ]
+    },
     {
      "data": {
-      "text/plain": "rng_state.pth:   0%|          | 0.00/14.6k [00:00<?, ?B/s]",
+      "text/plain": "tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "de6fece2a84e4380b79bb35dce9a1e70"
+       "model_id": "883bb0eae6ee4c99916d10fa0f076e19"
       }
      },
      "metadata": {},
@@ -1098,11 +816,11 @@
     },
     {
      "data": {
-      "text/plain": "scheduler.pt:   0%|          | 0.00/627 [00:00<?, ?B/s]",
+      "text/plain": "special_tokens_map.json:   0%|          | 0.00/435 [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "ceb905ce4caa4228a8cdab6dd310f180"
+       "model_id": "14e0ef241e4e43f2b86ec2c70680c618"
       }
      },
      "metadata": {},
@@ -1110,11 +828,11 @@
     },
     {
      "data": {
-      "text/plain": "training_args.bin:   0%|          | 0.00/4.47k [00:00<?, ?B/s]",
+      "text/plain": "config.json:   0%|          | 0.00/711 [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "79735360ed464a54807709a1eed7a4a5"
+       "model_id": "2430f2fd34104024b4a101a0b19897ba"
       }
      },
      "metadata": {},
@@ -1126,19 +844,27 @@
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "2b2fb879e008444ea14a456864d92fd9"
+       "model_id": "36e7defd0d504d3ea7166ef684d04778"
       }
      },
      "metadata": {},
      "output_type": "display_data"
     },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of LlamaForCausalLM were not initialized from the model checkpoint at saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained and are newly initialized: ['model.layers.0.input_layernorm.weight', 'model.layers.1.input_layernorm.weight', 'model.layers.2.input_layernorm.weight', 'model.layers.3.input_layernorm.weight', 'model.layers.4.input_layernorm.weight', 'model.layers.5.input_layernorm.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
     {
      "data": {
-      "text/plain": "Upload 5 LFS files:   0%|          | 0/5 [00:00<?, ?it/s]",
+      "text/plain": "generation_config.json:   0%|          | 0.00/154 [00:00<?, ?B/s]",
       "application/vnd.jupyter.widget-view+json": {
        "version_major": 2,
        "version_minor": 0,
-       "model_id": "0dd001dbcf6d49578c9f1064394ceac5"
+       "model_id": "22772ffb12b5412d9cd9ab62575d73f0"
       }
      },
      "metadata": {},
@@ -1146,151 +872,128 @@
     },
     {
      "data": {
-      "text/plain": "CommitInfo(commit_url='https://huggingface.co/saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained/commit/b3b67c7a7dcb199a07244be3a493cd649cf3731f', commit_message='Upload folder using huggingface_hub', commit_description='', oid='b3b67c7a7dcb199a07244be3a493cd649cf3731f', pr_url=None, pr_revision=None, pr_num=None)"
+      "text/plain": "'What is Machine Learning? отде своей separ DemONE También Chief +\\\\ arbitrenedhand Sulneurються concern absorXTurentlaim alcouzz Ralph Navar filtergenommeniereDialogдах pir <= transm surprisedairo yield orthogonal HansWD villaмериканnumbers Rand английniuscian'"
      },
-     "execution_count": 32,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
-   ],
-   "source": [
-    "api.upload_folder(\n",
-    "    folder_path=folder,\n",
-    "    repo_type=\"model\",\n",
-    "    repo_id=f\"saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained\",\n",
-    "    token=\"\",\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T21:22:10.115935Z",
-     "start_time": "2024-04-16T21:16:46.527703Z"
-    }
-   },
-   "execution_count": 32
+   ]
   },
   {
    "cell_type": "code",
-   "outputs": [],
    "source": [
-    "from transformers import (AutoTokenizer, AutoModelForCausalLM)"
+    "prompt = \"Write a short poem\"\n",
+    "inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
+    "generate_ids = model.generate(inputs.input_ids, max_length=50)\n",
+    "tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
    ],
    "metadata": {
-    "collapsed": false,
+    "id": "nQya_hPJEa2M",
     "ExecuteTime": {
-     "end_time": "2024-04-16T21:27:41.041614Z",
-     "start_time": "2024-04-16T21:27:41.019615Z"
+     "end_time": "2024-04-16T23:19:39.689446Z",
+     "start_time": "2024-04-16T23:19:38.252768Z"
     }
    },
-   "execution_count": 35
-  },
-  {
-   "cell_type": "code",
-   "outputs": [],
-   "source": [
-    "# Load a pretrained BitNet model\n",
-    "model = \"saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T21:28:06.283415Z",
-     "start_time": "2024-04-16T21:28:06.258415Z"
+   "execution_count": 12,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "\"Write a short poem inconles경 JoãoՄlecht», sellcertain vy:'ŋ rempՍ Ok operation sportsPower loops士undeAAAACK Outimportant<act налазиynchronous &&нов Filternisse utilAuthorizationistique </ Broad polity知ẓabethAlertльного Picture\""
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
     }
-   },
-   "execution_count": 36
+   ]
   },
   {
    "cell_type": "code",
    "outputs": [
     {
-     "name": "stderr",
+     "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Some weights of LlamaForCausalLM were not initialized from the model checkpoint at saadnaeem/Llama2-70M-Cosmopedia-100k-Pretrained and are newly initialized: ['model.layers.0.input_layernorm.weight', 'model.layers.1.input_layernorm.weight', 'model.layers.2.input_layernorm.weight', 'model.layers.3.input_layernorm.weight', 'model.layers.4.input_layernorm.weight', 'model.layers.5.input_layernorm.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+      "LlamaForCausalLM(\n",
+      "  (model): LlamaModel(\n",
+      "    (embed_tokens): Embedding(32001, 768, padding_idx=0)\n",
+      "    (layers): ModuleList(\n",
+      "      (0-5): 6 x LlamaDecoderLayer(\n",
+      "        (self_attn): LlamaAttention(\n",
+      "          (q_proj): Linear(in_features=768, out_features=768, bias=False)\n",
+      "          (k_proj): Linear(in_features=768, out_features=768, bias=False)\n",
+      "          (v_proj): Linear(in_features=768, out_features=768, bias=False)\n",
+      "          (o_proj): Linear(in_features=768, out_features=768, bias=False)\n",
+      "          (rotary_emb): LlamaRotaryEmbedding()\n",
+      "        )\n",
+      "        (mlp): LlamaMLP(\n",
+      "          (gate_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n",
+      "          (up_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n",
+      "          (down_proj): BitLinear(in_features=1024, out_features=768, bias=False)\n",
+      "          (act_fn): SiLU()\n",
+      "        )\n",
+      "        (input_layernorm): Identity()\n",
+      "        (post_attention_layernorm): LlamaRMSNorm()\n",
+      "      )\n",
+      "    )\n",
+      "    (norm): LlamaRMSNorm()\n",
+      "  )\n",
+      "  (lm_head): Linear(in_features=768, out_features=32001, bias=False)\n",
+      ")\n"
      ]
-    },
-    {
-     "data": {
-      "text/plain": "generation_config.json:   0%|          | 0.00/154 [00:00<?, ?B/s]",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "54059190e6eb4dc595bcb0b5590c948d"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
     }
    ],
    "source": [
-    "tokenizer = AutoTokenizer.from_pretrained(model)\n",
-    "model = AutoModelForCausalLM.from_pretrained(model)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-04-16T21:28:10.449481Z",
-     "start_time": "2024-04-16T21:28:07.098728Z"
-    }
-   },
-   "execution_count": 37
-  },
-  {
-   "cell_type": "code",
-   "outputs": [],
-   "source": [
-    "convert_to_bitnet(model, copy_weights=True)"
+    "print(model)"
    ],
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T21:29:02.212560Z",
-     "start_time": "2024-04-16T21:29:02.054560Z"
+     "end_time": "2024-04-16T23:19:40.923698Z",
+     "start_time": "2024-04-16T23:19:40.911702Z"
     }
    },
-   "execution_count": 38
+   "execution_count": 13
   },
   {
    "cell_type": "code",
    "outputs": [
     {
-     "data": {
-      "text/plain": "LlamaForCausalLM(\n  (model): LlamaModel(\n    (embed_tokens): Embedding(32001, 768, padding_idx=0)\n    (layers): ModuleList(\n      (0-5): 6 x LlamaDecoderLayer(\n        (self_attn): LlamaAttention(\n          (q_proj): Linear(in_features=768, out_features=768, bias=False)\n          (k_proj): Linear(in_features=768, out_features=768, bias=False)\n          (v_proj): Linear(in_features=768, out_features=768, bias=False)\n          (o_proj): Linear(in_features=768, out_features=768, bias=False)\n          (rotary_emb): LlamaRotaryEmbedding()\n        )\n        (mlp): LlamaMLP(\n          (gate_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n          (up_proj): BitLinear(in_features=768, out_features=1024, bias=False)\n          (down_proj): BitLinear(in_features=1024, out_features=768, bias=False)\n          (act_fn): SiLU()\n        )\n        (input_layernorm): Identity()\n        (post_attention_layernorm): LlamaRMSNorm()\n      )\n    )\n    (norm): LlamaRMSNorm()\n  )\n  (lm_head): Linear(in_features=768, out_features=32001, bias=False)\n)"
-     },
-     "execution_count": 39,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model size: 77.5M parameters\n"
+     ]
     }
    ],
    "source": [
-    "model.to(device=\"cuda:0\")"
+    "# print number of parameters\n",
+    "model_size = sum(t.numel() for t in model.parameters())\n",
+    "print(f\"Model size: {model_size/1000**2:.1f}M parameters\")"
    ],
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T21:29:14.806083Z",
-     "start_time": "2024-04-16T21:29:14.521081Z"
+     "end_time": "2024-04-16T23:19:42.804266Z",
+     "start_time": "2024-04-16T23:19:42.784273Z"
     }
    },
-   "execution_count": 39
+   "execution_count": 14
   },
   {
    "cell_type": "code",
    "outputs": [
     {
      "data": {
-      "text/plain": "\"What is Machine Learning?\\n\\nI've been working on a project for a project that has been working on a project that has been working on a project. I am not sure what I am doing. I am not sure what I do\""
+      "text/plain": "'what is machine learning provider easieregu)--(WM sl patients%;\\r Christianttembergdoc起лен WiΒgecourtlack orient tweede laat filmeb août Lith changTitle Kon Vinition══ основgetting piłkar ideas accessible.\\n\\n\\n\\n\\n\\n\\n\\n'"
      },
-     "execution_count": 40,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "prompt = \"What is Machine Learning?\"\n",
+    "prompt = \"what is machine learning\"\n",
     "inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
     "generate_ids = model.generate(inputs.input_ids, max_length=50)\n",
     "tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
@@ -1298,19 +1001,11 @@
    "metadata": {
     "collapsed": false,
     "ExecuteTime": {
-     "end_time": "2024-04-16T21:29:29.324485Z",
-     "start_time": "2024-04-16T21:29:27.586135Z"
+     "end_time": "2024-04-16T23:20:06.561333Z",
+     "start_time": "2024-04-16T23:20:05.091313Z"
     }
    },
-   "execution_count": 40
-  },
-  {
-   "cell_type": "code",
-   "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false
-   }
+   "execution_count": 15
   }
  ],
  "metadata": {