{"cells":[{"cell_type":"markdown","source":["# **Language Translation**"],"metadata":{"id":"MY5EA3RRGumY"}},{"cell_type":"markdown","source":["This is the code of language translation from english to french. I have done a fine tuning of a dataset taken from huggingface on a pre trained model from huggingface only. The details of each cell code is given above that cell or in comments. "],"metadata":{"id":"sQhvGL7IHsY_"}},{"cell_type":"markdown","source":["First we will connect the GPU so as to run our program. GPU is very useful to run programs which have very large datasets."],"metadata":{"id":"tgdkWb8zG-cF"}},{"cell_type":"code","source":["!nvidia-smi #checking whether GPU is working or not"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Fet3K5GfHHpQ","executionInfo":{"status":"ok","timestamp":1692193558563,"user_tz":-330,"elapsed":10,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}},"outputId":"bb5bb8f2-7e6f-434f-e3fa-369833caec8b"},"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Wed Aug 16 13:46:04 2023 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","| | | MIG M. |\n","|===============================+======================+======================|\n","| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n","| N/A 65C P8 12W / 70W | 0MiB / 15360MiB | 0% Default |\n","| | | N/A |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: |\n","| GPU GI CI PID Type Process name GPU Memory |\n","| ID ID Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"]}]},{"cell_type":"code","source":["! pip install -q transformers accelerate sentencepiece gradio datasets evaluate sacrebleu\n","#installing all the necessary libraries"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UF0lJfd5HFXs","executionInfo":{"status":"ok","timestamp":1692193585491,"user_tz":-330,"elapsed":26932,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}},"outputId":"c38b1903-ec56-4b97-f2bd-c73f1cffd7d9"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m32.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.0/20.0 MB\u001b[0m \u001b[31m63.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m49.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m86.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m74.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.8/65.8 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.4/297.4 kB\u001b[0m \u001b[31m33.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.3/140.3 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.5/46.5 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.5/87.5 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m74.5/74.5 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n"]}]},{"cell_type":"markdown","source":["From the datasets download the dataset kde4(available on huggingface) which is a sample dataset for languuage translation and mention the two languages."],"metadata":{"id":"-qSxRNqSIk_2"}},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":281,"referenced_widgets":["22bcd0de966d4cb5bac337af42d66e0a","00969734af0a483bb653388c03371e30","690e162651c647a48401f2116d9c0a87","676c7aa802224adbb84701fa267b9ac7","cfed60dd3ed74bb7b48cacc0fd1050c2","dd5fcb2eccba468fa61d9bec549ff566","294935b56ac74d17b912a92c4d986a0e","3ee786d6fd8a488d8e85bbd1a7db33ab","84a80f90fa274d77b67ef23dd8ab667c","0b365fd4f68945de8537abc728a17a2a","5d4f056e3b2a433ca097adc9f27742af","1d343cc64ac14e24b7fb8be89dce7b81","d991be5c17e3463f9594361134992112","3568ff949a244718b2664595bb00f558","98a088bc290d4e4ebc8f2c789a1bb18d","8dfe395f9a39412f86cdaaec7e0e4c75","f2d6b74df93b4966b570c902bef631cc","74032a1241fe4f1fbcb45e727827f13f","148252f08c4048d29c2dc8decc00117b","db954ce91fba47338913ec58ea37efbc","3fd124d8e7684e178836533a58529e0e","dd0f75907ebe4fe2afa2f8a5b948641c","9b09b999d6a449ac8d1f89aad3c8edbd","d73b40fa1e5448d786a0586bb92af525","3e5918d902d34bb1b46a2d7ecb0b0903","0863747d6a494fafbcd2a868853a2fed","3f6bd35cad0e464b883c1d44f00acc32","e37a9b756c6f42a099cad27593fc8666","bb52f273142b4ddc854f3c84b64d23ce","1b8ed6cfdafe4359bdee4ec8c7f975e7","cbf8dcb1e90d4e1381a0b8c95531e210","280f89db66534fc2a835bd507e063759","eec5d4273bd64ac787d9416feb2e67d9","89e8cafe95a64d5b8ba839c5dd7f9ac9","28cb6c3155e8485ab421371201404ac9","747003774b6e406eb851f0058e403eb0","2d03f541abd94b26bdd82e713427d8c0","afa81bdfc0dd4532b7758ec6e39f8322","45bc50c516d749deb43cdd082c5ee1c2","db361ce12736438d86a6d46420853b69","396bfeeaede0475e94576f8d66497ed5","bda1922c4935447bbfb2f639d577bbb5","94ed17e677b14ab288412129d10155da","f9f69b32069948a4a9e0db5b4b5718c0","b766fdc8956b4a9fa49818f715fe4d47","542f280b944c4942b9bb69defd894014","f159b988fd634469b2b319b5285584d6","74c907575dd24ba9b3e83fe1a7ecf7ef","4560693d445945148727bbaabef7d449","4a48a7648e374b68878a719044f5e115","cb52a841e10e4f4b8fca5ac8de94d723","88ae98f9a4574f338b73345185ed0db5","40484d37eff54d699a89e19e663ca2a5","276095aa516b451aaac8f4f2fbbd9095","084f913d48724465a0d8c4886d8bb2fb"]},"executionInfo":{"elapsed":17707,"status":"ok","timestamp":1692193603192,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"NRnwM8bcnmGP","outputId":"127a2203-c256-4bff-8668-e5d0bad14c5f"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading builder script: 0%| | 0.00/4.25k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"22bcd0de966d4cb5bac337af42d66e0a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading metadata: 0%| | 0.00/8.45k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1d343cc64ac14e24b7fb8be89dce7b81"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading readme: 0%| | 0.00/5.10k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9b09b999d6a449ac8d1f89aad3c8edbd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading data: 0%| | 0.00/7.05M [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"89e8cafe95a64d5b8ba839c5dd7f9ac9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating train split: 0%| | 0/210173 [00:00, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b766fdc8956b4a9fa49818f715fe4d47"}},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["DatasetDict({\n"," train: Dataset({\n"," features: ['id', 'translation'],\n"," num_rows: 210173\n"," })\n","})"]},"metadata":{},"execution_count":3}],"source":["from datasets import load_dataset\n","raw_datasets = load_dataset(\"kde4\", lang1=\"en\", lang2=\"fr\")\n","raw_datasets"]},{"cell_type":"markdown","source":["In the original dataset we have only train set but we want both train and test sets to compare our outputs so use train_test_split function to break the dataset into two sets train and test."],"metadata":{"id":"LJfI3lcrJJcm"}},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1688,"status":"ok","timestamp":1692193604860,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"wJ4OnmL0n1YF","outputId":"a5399434-012f-4410-b0d5-65f073c98cf7"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["DatasetDict({\n"," train: Dataset({\n"," features: ['id', 'translation'],\n"," num_rows: 168138\n"," })\n"," test: Dataset({\n"," features: ['id', 'translation'],\n"," num_rows: 42035\n"," })\n","})"]},"metadata":{},"execution_count":4}],"source":["from sklearn.model_selection import train_test_split\n","split_datasets= raw_datasets[\"train\"].train_test_split(test_size=0.2, seed=20)\n","split_datasets"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":8,"status":"ok","timestamp":1692193604860,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"x3oI6dujodIT","outputId":"45502a31-607b-47d3-d856-c28c1cb0301b"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'en': 'If you choose the wrong settings here your articles could be unreadable or not sendable at all, so please be careful with these settings.',\n"," 'fr': 'Si vous choisissez ici les mauvais paramètres, vos articles peuvent devenir illisibles ou vous ne pourrez pas du tout les envoyer. Veuillez donc être prudent avec ces paramètres.'}"]},"metadata":{},"execution_count":5}],"source":["split_datasets[\"train\"][45][\"translation\"]"]},{"cell_type":"markdown","source":["Now we will be using a pre trained model from HuggingFace Helsinki-NLP/opus-mt-en-fr which contained the pre trained model of translation from english to french."],"metadata":{"id":"cDdD0lxUJg5K"}},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":310,"referenced_widgets":["56e528c8feb24f4bb1bbf3b241ee5710","cbccbc77fbfb4508b3ac83197d1b88da","b0ae81435c7d4d45904d88835cb40a82","4c3a2993818544a39019c7337ce97bcc","280496436450440692e6c5ecc376082b","bf790403f2214d2daea852f079c0a2c0","bb19d5990b1b44149afedcc287d9cebf","63cbc6fe25414a6197c565e4942e4833","08f8622003454668abad4b0ac7f9eeaa","6affdd0fa5464ae79a1227ec155d18f9","679c3405fd15414d817da839226265d3","05b0dcccd3a64ba098d3922374da6de6","8b6668dab5544aeb9ea8b6e225df97ab","1044be863f8f4e80823fbebc74d0d936","47b8af205f6c4b8794e3bff3078fc7fd","726c7ef4245842d7bfc28241413f30db","fd88caf04ebe472f949d70efd9feee62","94e43bad257341f3ba59264004293661","faafd5a9257449abad8a55b744d279f3","a6bb1f88d94e4568a44b6a307cfc8957","508a0eef37dd4e5abc2f2e4f6e1eef6f","91e65d55080141329dcbf9ecdff64992","6fc39feaabe84dafb275db101ae02009","d3bce2c662974ee0822b889faa5cbbf6","8f8a80a830464491bf876e0d0e5a67f0","4cec373011954a0cb13ff9abfc00951e","a57edf53c26c427a8f1c9dd713a9c698","b95f90ff4aa34a5682451802167490a8","e07b08f90e6b4e5bba94afab82ba0a51","a639b23e7b84432da291443a64138c71","8ecc35de20a64872b15fd6a3e51e09bf","3b31f9b2ef7843f4b0091da6a32468bb","3a27f449f4364f7e98195832c1dc9c92","a1cc6b28dba6498e96f5b2c72036ca79","fdf3b8f8e9ac4131841f79538e050adc","62421cc3c20044f684fd08bc2aa5b786","7e949de543244c25b153f09679399ef0","959d1ef3808244c88f8fc0bc48144acd","225e47d2077e471bbf353778805b50e0","7ae6ccd1dcaa4821bd2c56e7b92a1db3","9f4b94d33e2d48e39f2c89d5dd010946","142ace3cddb04252b77a8b09d349fd09","8d9d6ac498724578944e7cd909f1270c","f26b172629134fce968984fdb15e4a4d","6e16d16484944cae901a286254cc6bcf","0a14d65bd24e48b2a1afa1ba487a2445","7ffcd9872e934c5883a3098b144658c3","be5b53c878ee4db09c9d7cbf1862e02f","ccc1c3ed16e1482abf2c724aee394106","ecf063ce3c1946a785c99dd7bd454cbd","7785f900ef934a4a8bc256b6c0613fd4","9158922eee8741de9e5bf898f2b5fbf1","750b94df3d54497b8c1688b66bc2763a","4be4169cec1944a8bc9ab4e2dc208087","6e56eea531224d40a78a07e9d89eda65","e524c14360a04952917db8417dfa612a","e16cb488e58b4b46b20c0f9fa5c6ca02","f91aed5655224b8f98aef3832cf68bbf","d3b80daded7347958f0ae521e55ff494","843704debd674a448e9860311c28fe1c","d607f1e045cb4f8eb45db7515c4ab32e","d0a6a5b8625c4f268a37210941e15bfa","0edfcfef766442dfbb95e7d680322b21","b0438dd323c7453b8065f9470f437b06","41536c5b41494f3f8bb752c1494d14fc","19029d1b4e7e4131be69f83149a4b424","1c76fb860d6d4cfcbd4b174e76034f3f","f3bf210f767e4031b3741745c3c7e627","690bb16d91ed40d3b0c3b8b145eafdf9","31488007c90e4f3f8d67202b1533b991","ece5e567ed1a4225983d2fae63eea4bd","7a9fe2f0c0c3418c92c96e23537a14e8","1f8188acc075424f93caccf123d87b0e","bdf9314dc2e84362895d22f839e8f729","df4cdc1ddfa243a493b4ce080511582e","8f948aee70574dd0a6066f88156b9371","a0863a12c92f4290ad649f0091798f53"]},"executionInfo":{"elapsed":26601,"status":"ok","timestamp":1692193631457,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"X1yfmRCGo41_","outputId":"e68a2419-6eeb-48f9-df2c-66fc1c9f4a73"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading (…)lve/main/config.json: 0%| | 0.00/1.42k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"56e528c8feb24f4bb1bbf3b241ee5710"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading pytorch_model.bin: 0%| | 0.00/301M [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"05b0dcccd3a64ba098d3922374da6de6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)neration_config.json: 0%| | 0.00/293 [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6fc39feaabe84dafb275db101ae02009"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)okenizer_config.json: 0%| | 0.00/42.0 [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a1cc6b28dba6498e96f5b2c72036ca79"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)olve/main/source.spm: 0%| | 0.00/778k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6e16d16484944cae901a286254cc6bcf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)olve/main/target.spm: 0%| | 0.00/802k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e524c14360a04952917db8417dfa612a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)olve/main/vocab.json: 0%| | 0.00/1.34M [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1c76fb860d6d4cfcbd4b174e76034f3f"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/transformers/models/marian/tokenization_marian.py:194: UserWarning: Recommended: pip install sacremoses.\n"," warnings.warn(\"Recommended: pip install sacremoses.\")\n"]},{"output_type":"execute_result","data":{"text/plain":["[{'translation_text': \"Si vous choisissez les mauvais paramètres ici, vos articles pourraient être illisibles ou ne pas être envoyés du tout, alors s'il vous plaît soyez prudent avec ces paramètres.\"}]"]},"metadata":{},"execution_count":6}],"source":["from transformers import pipeline\n","model=\"Helsinki-NLP/opus-mt-en-fr\"\n","translator=pipeline(\"translation\", model=model)\n","translator(\"If you choose the wrong settings here your articles could be unreadable or not sendable at all, so please be careful with these settings.\")"]},{"cell_type":"markdown","source":["We can see that the pre trained model itself is giving us somewhat good translation and after fien tuning it will get better."],"metadata":{"id":"3Qi73zBYJ8Rb"}},{"cell_type":"markdown","source":["Now use the AutoTokenizer to use the same tokenizer which is used in pre trained model on the dataset we are using."],"metadata":{"id":"EC1cwrWTKIB_"}},{"cell_type":"code","execution_count":7,"metadata":{"id":"kSNwMIrNsN0U","executionInfo":{"status":"ok","timestamp":1692193631796,"user_tz":-330,"elapsed":357,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}}},"outputs":[],"source":["from transformers import AutoTokenizer\n","tokenizer=AutoTokenizer.from_pretrained(model, return_tensors=\"pt\")"]},{"cell_type":"code","execution_count":8,"metadata":{"id":"_yokqgMF1HBS","executionInfo":{"status":"ok","timestamp":1692193631797,"user_tz":-330,"elapsed":4,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}}},"outputs":[],"source":["def pre_processtext(text):\n"," inputs=[sample['en'] for sample in text['translation']]\n"," output=[sample['fr'] for sample in text['translation']]\n"," tokenized_text=tokenizer(inputs, text_target=output, max_length=128, truncation=True) #(text_target because if not done it will tokenize the french sentence according to english and so the labels will then not be correct)\n"," return tokenized_text"]},{"cell_type":"markdown","source":["Now we will tokenize the whole dataset in one go using the map function."],"metadata":{"id":"sb0R2d67KP3C"}},{"cell_type":"code","execution_count":9,"metadata":{"id":"bp6Sv37y2AYm","colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["29b4ad03b52a4deca9c25c07afcc0577","76756c4a463e491e93ef99d08abc5411","c5bf574c411442e3a20350eaa30d1a59","b835d7db170c43b28d78618e7e0ae655","9abd9164c72d4686944057e577509236","6f96ca86155646afb84cf6b073e61fef","5ab8eb72777848e0a4c3774240a56c1e","ca02672e957b4c60a0a59ff91bced685","49a395142e674fd3998a78d1aea49533","911f0f3b916646ba885840902e107b61","6643410a92c34b81b471184a5bc25e62","e3a37d21788d4723a048bcc02a980730","1bed9e17d66c4324bb8689613d2bc80a","3c36335007c847b1be732dfe6c27c3c0","731783a4c23d4290897cf59c48786f6f","40b8808d6fde4738b3564974ece83962","eaa1aad8e1e8485797575e97e402918f","66c2bc7ccd354d90a3d6fa7bc3427b91","9ea8a93f4aab4803a003808b21a459ee","68ac1a0e290045a7a085665b40d9a097","d86c6fa6cf444d64990f73300217783e","f90dff7cdf594d48a98b0786c892114f"]},"executionInfo":{"status":"ok","timestamp":1692193692356,"user_tz":-330,"elapsed":60563,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}},"outputId":"d00cdab8-260f-4a78-f037-1b19c92e938c"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/168138 [00:00, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"29b4ad03b52a4deca9c25c07afcc0577"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/42035 [00:00, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3a37d21788d4723a048bcc02a980730"}},"metadata":{}}],"source":["tokenized_datasets=split_datasets.map(\n"," pre_processtext,\n"," batched=True,\n"," remove_columns=split_datasets[\"train\"].column_names #(to remove extra columns)\n",")"]},{"cell_type":"markdown","source":["Now after preprocessing we have to choose a model to train and we will be using the AutoModelForSeq2SeqLM"],"metadata":{"id":"NHWsFEfRK5QJ"}},{"cell_type":"code","execution_count":10,"metadata":{"id":"LE8WLR177Lv-","executionInfo":{"status":"ok","timestamp":1692193695060,"user_tz":-330,"elapsed":2721,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}}},"outputs":[],"source":["from transformers import AutoModelForSeq2SeqLM\n","model_1= AutoModelForSeq2SeqLM.from_pretrained(model)"]},{"cell_type":"markdown","source":["Data collator is also an important tool which is used for dynamic padding and adding -100 to short sentences to make it to match max length and also adding a start of sentence token which can be seen in decoder_input_ids."],"metadata":{"id":"61NDGQXwLLyw"}},{"cell_type":"code","execution_count":11,"metadata":{"id":"KCehvmtU4Wd0","executionInfo":{"status":"ok","timestamp":1692193695060,"user_tz":-330,"elapsed":6,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}}},"outputs":[],"source":["from transformers import DataCollatorForSeq2Seq\n","data_collator=DataCollatorForSeq2Seq(tokenizer,model=model_1)"]},{"cell_type":"code","execution_count":12,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1692193695060,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"eNs1nUEq5ddQ","outputId":"08604315-26f0-4618-a7e4-a481a96af581"},"outputs":[{"output_type":"stream","name":"stdout","text":["dict_keys(['input_ids', 'attention_mask', 'labels', 'decoder_input_ids'])\n","tensor([[25966, 19, 540, 8, 669, 33355, 24, 11106, 37, 583,\n"," 583, 9507, 10571, 3, 49, 19015, 3, 49, 1937, 74,\n"," 2635, 973, 529, 13518, 74, 102, 0],\n"," [14743, 301, 548, 0, -100, -100, -100, -100, -100, -100,\n"," -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n"," -100, -100, -100, -100, -100, -100, -100]])\n"]},{"output_type":"execute_result","data":{"text/plain":["tensor([[59513, 25966, 19, 540, 8, 669, 33355, 24, 11106, 37,\n"," 583, 583, 9507, 10571, 3, 49, 19015, 3, 49, 1937,\n"," 74, 2635, 973, 529, 13518, 74, 102],\n"," [59513, 14743, 301, 548, 0, 59513, 59513, 59513, 59513, 59513,\n"," 59513, 59513, 59513, 59513, 59513, 59513, 59513, 59513, 59513, 59513,\n"," 59513, 59513, 59513, 59513, 59513, 59513, 59513]])"]},"metadata":{},"execution_count":12}],"source":["batch = data_collator([tokenized_datasets[\"train\"][i] for i in range(1,3)])\n","print(batch.keys())\n","print(batch['labels'])\n","batch['decoder_input_ids']"]},{"cell_type":"markdown","source":["For evaluating our model we will be using sacrebleu score which basically doesnt check the sentences grammatically but they see how many words are matching with each other and also penalise if the same word is coming multiple times but its not their in the original translation."],"metadata":{"id":"5vwR9WbxLfvi"}},{"cell_type":"code","execution_count":13,"metadata":{"id":"0I9_9UOs9-CV","colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["f12d2c8feec24e718fdd7dc183101fb9","96d422a557474e32af9850f95dbec78f","ca7ae960ccda49dcb2509440e91cdf69","12e5b3a94bfe496bb61200fb493ba46d","2a86a348bc174429a9dd7c106d8eedbd","474ee349f59945aaad820a918bafe875","2c533699e74f4508905ee043f7287a8d","1909157fc0f04e5ca13d94983892ae65","a023514a70ed41febda596d926a63d8a","5e7f67f8da6d4a7ba7054e1862ddcbc6","48530a501020493e89cf68ef6dcb4020"]},"executionInfo":{"status":"ok","timestamp":1692193696095,"user_tz":-330,"elapsed":1038,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}},"outputId":"90c38e94-730b-41be-d398-b67ba1544535"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading builder script: 0%| | 0.00/8.15k [00:00, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f12d2c8feec24e718fdd7dc183101fb9"}},"metadata":{}}],"source":["import evaluate\n","metric_evaluate= evaluate.load(\"sacrebleu\")"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"Lgo74zb7-9ds","executionInfo":{"status":"ok","timestamp":1692193696095,"user_tz":-330,"elapsed":10,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"}}},"outputs":[],"source":["import numpy as np\n","\n","def compute_metrics(eval):\n"," preds, labels= eval\n"," if isinstance(preds, tuple): #if model returns more than the prediction logits\n"," preds=preds[0]\n"," decoded_preds= tokenizer.batch_decode(preds, skip_special_tokens=True)\n","\n"," labels=np.where(labels != -100, labels,tokenizer.pad_token_id) #replacing -100 as we will not be able to decode them\n"," decoded_labels=tokenizer.batch_decode(labels, skip_special_tokens=True)\n","\n"," decoded_preds=[pred.strip() for pred in decoded_preds]\n"," decoded_labels=[[label.strip()] for label in decoded_labels] #references should be list of list of sentences\n","\n"," result=metric_evaluate.compute(predictions=decoded_preds, references=decoded_labels)\n"," return {\"bleu\": result[\"score\"]}\n"]},{"cell_type":"markdown","source":["I will be storing my model in hugging face repository so login to HuggingFace"],"metadata":{"id":"hVC-1fpLMBXY"}},{"cell_type":"code","execution_count":15,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":145,"referenced_widgets":["613c57cb99844be0a91b42bbbce3a4fa","a0d4742034e0432491bb118ad5c4b284","3242055d6c9a4d98b88281d1ccd30aaf","75b5786c560a4bf6a038d46692c2bc14","f24df78c0332409d81732570a29dd07d","675661d849e1427dab1ae9fcea466144","39cfc64e667c4588b6c588e8089bedd4","3d7b635dd56540b7b55ba86545e8d8f7","a566cf1e606c4d57bc1619c172911457","0929292d5d694ad181b5c790af0f741e","bf7ec76ff6494bef8ab231feacf6f03d","87adcd3b02cf4d839f333378c1ff82d4","3de20f95870645db86ce3f28f8f0f382","e8df1081f9ec41d4a4d3dd72b5fb67e3","58d543e159a644c3a0c98768b8fb219e","f9cc829598cd4eb483ec098f3284d95b","fbd5f58dd3be4dccbf150e328e641f39","4a12bd5ba620474ead77fb8f4563930a","b93513639f4e44899310cd9354711186","df9d2f0ed38b4143bdf30d798af2311b","e4758624a569408087524dcd9b99947c","1549f486948f4000a6ef3a10bfc5273c","19c613bc714a4bdcb27ad1c555f7b7bf","e2bf0014166743a2ad23db3d901f198c","45c88b7972ff43ce8e9b8d096a184d69","2fa9d2b0fb0c4fb6863cde3ade4380d0","99dd0e70c55a43abaa0fbb1ee84bea35","3f3dca9f4b9c4f7b9601c9e4cc038950","550fc1868fe2444a817f24b0c54f32f6","ef2934dd500a4d2ca9fd45b17984b586","9c840e876e1c424c9600e149261a33e0","f6faaab7951b4cea91fafce2336fee3f"]},"executionInfo":{"elapsed":10,"status":"ok","timestamp":1692193696096,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"Z2br-fqVBA5r","outputId":"1533946c-ce87-4028-e212-d8cd8e09facd"},"outputs":[{"output_type":"display_data","data":{"text/plain":["VBox(children=(HTML(value='
Step | \n","Training Loss | \n","
---|---|
500 | \n","1.365600 | \n","
1000 | \n","1.226100 | \n","
1500 | \n","1.155900 | \n","
2000 | \n","1.108200 | \n","
2500 | \n","1.084700 | \n","
3000 | \n","1.032800 | \n","
3500 | \n","1.020600 | \n","
4000 | \n","1.009300 | \n","
4500 | \n","0.994800 | \n","
5000 | \n","0.972400 | \n","
5500 | \n","0.920800 | \n","
6000 | \n","0.868800 | \n","
6500 | \n","0.847700 | \n","
7000 | \n","0.862200 | \n","
"],"text/plain":[" "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=15765, training_loss=0.9011330519560364, metrics={'train_runtime': 3145.1124, 'train_samples_per_second': 160.38, 'train_steps_per_second': 5.013, 'total_flos': 1.008207288336384e+16, 'train_loss': 0.9011330519560364, 'epoch': 3.0})"]},"metadata":{},"execution_count":19}],"source":["trainer.train() #(training starts form here)"]},{"cell_type":"code","execution_count":20,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":141},"executionInfo":{"elapsed":3198647,"status":"ok","timestamp":1692200168657,"user":{"displayName":"Suyash Shirish Gahankari","userId":"03254191982788877845"},"user_tz":-330},"id":"g9mEJ1uH3KED","outputId":"dcaf8a09-b42f-41a1-cdd4-0291b147b9af"},"outputs":[{"output_type":"display_data","data":{"text/plain":["\n"," \n","
\n"," \n"," \n"," \n"," Step \n"," Training Loss \n"," \n"," \n"," 500 \n"," 1.365600 \n"," \n"," \n"," 1000 \n"," 1.226100 \n"," \n"," \n"," 1500 \n"," 1.155900 \n"," \n"," \n"," 2000 \n"," 1.108200 \n"," \n"," \n"," 2500 \n"," 1.084700 \n"," \n"," \n"," 3000 \n"," 1.032800 \n"," \n"," \n"," 3500 \n"," 1.020600 \n"," \n"," \n"," 4000 \n"," 1.009300 \n"," \n"," \n"," 4500 \n"," 0.994800 \n"," \n"," \n"," 5000 \n"," 0.972400 \n"," \n"," \n"," 5500 \n"," 0.920800 \n"," \n"," \n"," 6000 \n"," 0.868800 \n"," \n"," \n"," 6500 \n"," 0.847700 \n"," \n"," \n"," 7000 \n"," 0.862200 \n"," \n"," \n"," 7500 \n"," 0.847000 \n"," \n"," \n"," 8000 \n"," 0.845700 \n"," \n"," \n"," 8500 \n"," 0.854800 \n"," \n"," \n"," 9000 \n"," 0.845600 \n"," \n"," \n"," 9500 \n"," 0.847600 \n"," \n"," \n"," 10000 \n"," 0.847900 \n"," \n"," \n"," 10500 \n"," 0.834000 \n"," \n"," \n"," 11000 \n"," 0.762300 \n"," \n"," \n"," 11500 \n"," 0.764200 \n"," \n"," \n"," 12000 \n"," 0.767200 \n"," \n"," \n"," 12500 \n"," 0.764900 \n"," \n"," \n"," 13000 \n"," 0.770000 \n"," \n"," \n"," 13500 \n"," 0.756800 \n"," \n"," \n"," 14000 \n"," 0.756600 \n"," \n"," \n"," 14500 \n"," 0.762600 \n"," \n"," \n"," 15000 \n"," 0.766800 \n"," \n"," \n"," \n","15500 \n"," 0.750800 \n","
Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.