{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "kWcrILQkt1eT" }, "source": [ "**RUMI-TO-JAWI TRANSLATION MODEL**\n", "\n", "References:\n", "https://github.com/ken11/mbart-finetuning/blob/master/mbart-finetuning.ipynb" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Sa0Obt8Kxm3K", "outputId": "1e51bb55-34e7-48e0-d356-fdb10f2fe3a2" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mounted at /content/drive\n" ] } ], "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')" ] }, { "cell_type": "markdown", "metadata": { "id": "bX6-74WYFgtj" }, "source": [ "# **Latest Code**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "collapsed": true, "id": "Zl0v7v5lQyL9", "outputId": "3d66ce67-dcc0-498f-ae73-9bc407c6206d" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.47.1)\n", "Collecting datasets\n", " Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.26.4)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.2.2)\n", "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.2.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\n", "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.2)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.10.0)\n", "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.1)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.27.0)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.11.6)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.21.0)\n", "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.67.1)\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess<0.70.17 (from datasets)\n", " Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec (from torch)\n", " Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.11.10)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.4)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.2)\n", "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.3.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (0.2.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.18.3)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.12.14)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.2)\n", "Downloading datasets-3.2.0-py3-none-any.whl (480 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: xxhash, fsspec, dill, multiprocess, datasets\n", " Attempting uninstall: fsspec\n", " Found existing installation: fsspec 2024.10.0\n", " Uninstalling fsspec-2024.10.0:\n", " Successfully uninstalled fsspec-2024.10.0\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0mSuccessfully installed datasets-3.2.0 dill-0.3.8 fsspec-2024.9.0 multiprocess-0.70.16 xxhash-3.5.0\n" ] } ], "source": [ "!pip install torch transformers datasets numpy pandas sentencepiece" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "collapsed": true, "id": "TCz_YdwCC7ql", "outputId": "939661b6-d202-45ac-bba0-c2b333816d36" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: fairseq==0.6.2 in /usr/local/lib/python3.10/dist-packages (0.6.2)\n", "Requirement already satisfied: cffi in /usr/local/lib/python3.10/dist-packages (from fairseq==0.6.2) (1.17.1)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from fairseq==0.6.2) (1.26.4)\n", "Requirement already satisfied: sacrebleu in /usr/local/lib/python3.10/dist-packages (from fairseq==0.6.2) (2.4.3)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from fairseq==0.6.2) (4.67.1)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi->fairseq==0.6.2) (2.22)\n", "Requirement already satisfied: portalocker in /usr/local/lib/python3.10/dist-packages (from sacrebleu->fairseq==0.6.2) (3.0.0)\n", "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu->fairseq==0.6.2) (2024.11.6)\n", "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu->fairseq==0.6.2) (0.9.0)\n", "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from sacrebleu->fairseq==0.6.2) (0.4.6)\n", "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu->fairseq==0.6.2) (5.3.0)\n" ] } ], "source": [ "!pip install fairseq==0.6.2" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gq00b5EuRhrN" }, "outputs": [], "source": [ "from transformers import MBartForConditionalGeneration, MBartTokenizer, MBartConfig" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Xa26mqdTTh9R" }, "outputs": [], "source": [ "from datasets import load_from_disk" ] }, { "cell_type": "markdown", "metadata": { "id": "7ruBN-P2GvaE" }, "source": [ "**Preparing the Dataset**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nbJ0C7n7Gym_" }, "outputs": [], "source": [ "from datasets import load_dataset\n", "dataset = load_dataset(\"mesolitica/rumi-jawi-instructions\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "FfFoUXn5G0ct", "outputId": "678e9da9-7376-4825-fba7-929d1024dca6" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['prompt_input', 'input', 'output'],\n", " num_rows: 1499594\n", " })\n", "})\n", "{'prompt_input': None, 'input': 'tukar ke rumi: بوين تنجوڠ مروڤاكن سبواه كچامتن د كابوڤاتين كاڤواس هولو، كليمنتن بارت، إندونيسيا.', 'output': 'Boyan Tanjung merupakan sebuah kecamatan di Kabupaten Kapuas Hulu, Kalimantan Barat, Indonesia.'}\n" ] } ], "source": [ "print(dataset)\n", "print(dataset['train'][0]) # Display the first sample" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7TbM_QPSG7dr" }, "outputs": [], "source": [ "dataset = dataset.remove_columns(['prompt_input'])" ] }, { "cell_type": "markdown", "metadata": { "id": "iG-_nYbQNBh0" }, "source": [ "**Need to take note here. The dataset should be splitted into two new datasets**\n", "\n", "This is because mesolitica datasets combined both 'tukar ke rumi' and 'tukar ke jawi' datasets.\n", "\n", "Which affecting the training process." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 162, "referenced_widgets": [ "5fbc47165db74f91b9e614df0d7df7fd", "e4edb8c36b374e20bf9531985a86b96b", "988b0139b0d8499da97cb7d3a5607ce4", "42f918171a8c4e66a59252c6dffb8a6a", "901560f4d7b7467288baa07c12c62f04", "aec7f152f8b74666a49a04e643244e1c", "429c06890a784aceb0623402b6fba0c9", "daefc42c94904556806e88b19edbcc77", "855af7723a93453e9bd9eef111ff8b34", "cccdef69b1c04df1870d868a711595b1", "5400ca171013472f8b8d6c15c3e86589", "b1d13ab11f664bd2a76d5f2cf1b17cff", "94e1781694124a8ba77539a435d062de", "6417468ac85b4f52b53b67ca9f7fe6e1", "61077ce93ce54a6291ae4c1e3f0188bd", "84c4b78940d24c9e82c24edfdf01a5fc", "04841423621f4c6ba9d257210a906ad5", "9944b8281bfd494ebf67cc72ff8f27a8", "0a012e2534d7482581c8667ba5497355", "52be4e574ef449ad85235b5bce17dcdd", "f0e7cb46d1a44e6ba127430311e663e3", "192ba7d635894855bd3463bdda35e0a3", "00e24c6f5f6b462387c4f41668145a6b", "c30dea49fc6d46388b028c2ae3b2078a", "2bc72303aba44d4c8fcd2fe25714b26b", "1f75d00957a6424686725b269c29c3f5", "26c1d1577ee243068a9e92e3d2ed7ef1", "9a8a0fc6c55c473e8ab7eb7db9dc34b4", "496f3711fc934d999c55486e6cd42ca2", "1595156d2bf541afbce127a3ad7eb986", "e92fbb2bad5048ccb18d275bc852771c", "04018a70f1ff4d3da242878e745aea66", "4b31f4aff6e941d18d5ebc55870d6951", "6a7ffce606a8400fa456ca2c22d834f1", "329046c64bf34965abce9e9354d5bd0d", "9bf19286853148c4bcd2bec31e5505d0", "a09cddb7290b4c2bb794cbf16b03e6ad", "61d50a37307e43dbb1ac181125bab487", "d9ab8a0801f74d35b749008f59cfe81a", "98516005540d42d68a9a5539c4c67e40", "268a7b7d7bc44775aad99e277ce7c105", "ebf425f5e3a24ce299e6bfdbfc4a0810", "5f6312c9836d4393a28ee379bd0e5a66", "3ead78b402d44bc1afa74b76c05b4c70" ] }, "id": "ZuibmsfEM_ym", "outputId": "4661f27b-6757-4941-e689-b79700ecd087" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5fbc47165db74f91b9e614df0d7df7fd", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Filter: 0%| | 0/1499594 [00:00] 5.23G 239MB/s in 24s \n", "\n", "2024-12-29 08:46:20 (222 MB/s) - ‘mbart.cc25.v2.tar.gz’ saved [5618826847/5618826847]\n", "\n", "mbart.cc25.v2/\n", "mbart.cc25.v2/sentence.bpe.model\n", "mbart.cc25.v2/dict.txt\n", "mbart.cc25.v2/model.pt\n", "dict.txt model.pt sentence.bpe.model\n" ] } ], "source": [ "!wget \"https://dl.fbaipublicfiles.com/fairseq/models/mbart/mbart.cc25.v2.tar.gz\"\n", "!tar -zxvf mbart.cc25.v2.tar.gz\n", "!ls mbart.cc25.v2" ] }, { "cell_type": "markdown", "metadata": { "id": "V9wXLcMwf0lm" }, "source": [ "**Formatting vocab**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yqa7-1KCf6bD" }, "outputs": [], "source": [ "edited = []\n", "for line in open(\"new_spm_model.vocab\", 'r', encoding='utf-8'):\n", " if line in [\"\\n\", \"\\n\", \"\\n\"]:\n", " continue\n", " new_line = line.rstrip('\\n') + \" 1\\n\"\n", " edited.append(new_line)\n", "\n", "with open('new_dict.txt', 'w') as f:\n", " for e in edited:\n", " f.write(e)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "collapsed": true, "id": "ZjUx8ROYKUQC", "outputId": "7ef916fc-b27a-4314-e502-61415433bbb5" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "V 1\n", "ث 1\n", "' 1\n", "Y 1\n", "ط 1\n", "ة 1\n", "ذ 1\n", "x 1\n", "Z 1\n", "q 1\n" ] } ], "source": [ "!tail new_dict.txt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "aq2aSqDxgBtw", "outputId": "073b016b-74a1-4f7c-ab1e-767ac9d6f6a1" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "drive\t mbart.cc25.v2.tar.gz new_spm_model.model output_dataset.txt split_dataset\n", "mbart.cc25.v2 new_dict.txt\t new_spm_model.vocab sample_data\t tmp.txt\n" ] } ], "source": [ "!ls" ] }, { "cell_type": "markdown", "metadata": { "id": "rBlARW8PgWNa" }, "source": [ "**Reduce to create new model**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IovbSfTngV32", "outputId": "b2bc82fe-4803-4e2e-e20c-a9190629cc96" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "drive\t\t new_dict.txt\t output_dataset.txt split_dataset\n", "mbart.cc25.v2\t new_spm_model.model reduced_model tmp.txt\n", "mbart.cc25.v2.tar.gz new_spm_model.vocab sample_data\n" ] } ], "source": [ "!mkdir reduced_model\n", "!ls" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "La-63DP1gfCs" }, "outputs": [], "source": [ "import numpy as np\n", "np.float = float\n", "\n", "from fairseq.data import Dictionary\n", "from transformers import (\n", " MBartForConditionalGeneration, MBartTokenizer, MBartConfig\n", ")\n", "from typing import List\n", "import torch" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "b44a23b1c635457f90cfcfacadfeaee8", "cf5126785d6d4fa99e3cefa410beea30", "0f28f16a797d4871b475db465a7dfe3c", "c72f82181b5c4d7495d8f1e0da173004", "7d65829cc25945cc956aee274f7c95cd", "263a8fd4ba4941c5aba79a6a14872a18", "f3777d69e44b4957a79c2348d471ca5e", "a514eb8ea31c4f46b948f18411221735", "c2ce61060d7a49afb5015b30845b61f8", "255e8806c47d4abab7aa41e393ba18e8", "2d317a350d7341f99175034ec4c52dcb", "e2bad09a6029485193f85a986c014c4c", "c433f05101694e3881d44b5a597077f2", "625c961e0a49419eac19edf2d7f70e5d", "0f4000463e064fcdb2f3c6023c4bbf74", "8db25c530b4c4f6185f9844dd7610c1e", "c6e70930a4f44fee9f9baaf86bef2692", "962ea77573d94458b6b02192d2415bda", "e5137a0d7d7a493eb22d9b3cdea4cb3f", "72c164cc14584029a7fca1f3c22f7e84", "05a7b2ada59c48c6b9774cc725fc854f", "6985a0c76d64485e8b8aa7c0a9383690", "dce93797096f4aaca2cec4a464150027", "77b7f243f8d543f7816da869ef5997e9", "29c3cf060bf64fb281346c9782b52b20", "889d0460eb82410e86fa90dd28c79a0b", "fd723c6c74f0412896fc9b63f359f799", "385552daaa104ba2b2c8966945a556f4", "69648088f19345b3973330b733aa3362", "45b237badb6b4368ae03a64b33c7ab45", "0f642650516f4e34a4371dd7d6c8ed23", "ab126ab44440479c88d638799e343ab2", "32ba466b02c14bdf90f97b14a587cafa" ] }, "collapsed": true, "id": "MlvsMv6-H5EN", "outputId": "938a76a1-7344-4c1c-c954-02494831c690" }, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co./settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "config.json: 0%| | 0.00/1.19k [00:00 Dictionary:\n", " d = Dictionary.load(path)\n", " for ll in langs:\n", " d.add_symbol(f\"[{ll}]\")\n", " d.add_symbol(\"\")\n", " d.add_symbol(\"\")\n", " return d\n", "\n", "\n", "pre_dict = load_dict(langs, \"./mbart.cc25.v2/dict.txt\")\n", "ft_dict = load_dict(langs, \"./new_dict.txt\")\n", "\n", "model = MBartForConditionalGeneration.from_pretrained(\"facebook/mbart-large-cc25\")\n", "org_sd = model.state_dict()\n", "resized_sd = model.state_dict()\n", "\n", "mapping: List[int] = []\n", "for i in range(len(ft_dict)):\n", " word = ft_dict[i]\n", " mapping.append(pre_dict.index(word))\n", "\n", "for name in [\"model.encoder.embed_tokens.weight\", \"model.decoder.embed_tokens.weight\", \"model.shared.weight\", \"lm_head.weight\"]:\n", " pre_tensor: torch.Tensor = org_sd[name]\n", " ft_tensor = torch.zeros(\n", " [len(ft_dict), 1024], dtype=pre_tensor.dtype, layout=pre_tensor.layout, device=pre_tensor.device,\n", " )\n", " for ft_i, pre_i in enumerate(mapping):\n", " ft_tensor[ft_i] = pre_tensor[pre_i]\n", " resized_sd[name] = ft_tensor\n", "resized_sd[\"final_logits_bias\"] = resized_sd[\"final_logits_bias\"][:, :len(ft_dict)]\n", "\n", "config = MBartConfig.from_pretrained(\"facebook/mbart-large-cc25\")\n", "config.vocab_size = len(ft_dict)\n", "print(config)\n", "new_model = MBartForConditionalGeneration.from_pretrained(None, config=config, state_dict=resized_sd)\n", "new_model.save_pretrained(\"./reduced_model\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "7ByBAMyALbhs", "outputId": "1fc4d518-71d3-4341-c2bd-6eac00b40de3" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "config.json generation_config.json model.safetensors\n" ] } ], "source": [ "!ls reduced_model" ] }, { "cell_type": "markdown", "metadata": { "id": "s4TDjAocLfSV" }, "source": [ "**Preparation of Tokenizer**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 150, "referenced_widgets": [ "ec7d5ea7dedf4cb48d1262620a3790bb", "5f731ca5ce7a49aca73fd0d1de61c644", "f95150ea42c648e68eeb5295cdefe1cd", "095375d20ee94ae586327ed987c94b47", "5367fd052adc49da817a9b5519da5778", "36f2a8ae25424c31a702b1e5b2f548ec", "1c62938e7f39440ea47483be4181bb33", "b1bb09fc8023467292ecbc307f7b4c94", "33d23b0dec2f45f2adae2707ad8138ca", "df0a74a9529c40e7a20b5e64af9c0cc8", "8433150bd5d146229ffc614565dce32e", "84ff9010c0274ad9a28ff85284bd9a77", "d3dd4a2187424f3381a90dec06aabd9f", "ee7ed503ce844ee9b043af1160a5ddfe", "72a8d36af5eb4243803e987a8b596d6c", "d788f100ecd643b8bb98eb2d7c8dd59f", "89efc9c3257142d78cc560833cb4c203", "fd47aa11cfac4921902ac80fe6702342", "68d8c441e87f40829ff1f5e5e89e6e82", "dc350db4006542e5aa101514842a00bf", "23589ab7e0284eec9f48487c8bb3bd70", "b4c291d5efa8429c8821dc87974bd701" ] }, "id": "YD9f0sDtLifW", "outputId": "2b37874e-3052-4bd1-c439-02bea3e02c2e" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "sentencepiece.bpe.model: 0%| | 0.00/5.07M [00:00" ], "text/html": [ "\n", "
\n", " \n", " \n", " [9373/9373 39:01, Epoch 1/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining LossValidation Loss

" ] }, "metadata": {} } ], "source": [ "trainer.train()\n", "trainer.save_model(result_dir)" ] }, { "cell_type": "markdown", "source": [ "**Debugging procedure for the Data_Collator**" ], "metadata": { "id": "3NPNzNUp1y7F" } }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "collapsed": true, "id": "c1SzW_uPsygP", "outputId": "00e804c3-d3d1-47e4-a8ba-613f91cffca4" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Debug Information ===\n", "Original MY: Euro 2020. Dia membuat penampilan sulung pada 2 Jun 2021 dalam perlawanan persahabatan menentang Wales, menggantikan Benjamin Pavard pada separuh masa pertama. Kehidupan\n", "Tokenized MY: [15090, 8365, 1662, 1715, 5375, 5210, 163, 141, 1917, 9029, 168, 2556, 13950, 3027, 8487, 63921, 4415, 27630, 21271, 1386, 163, 5805, 1104, 786, 63916, 5042, 2, 64004, 1, 1, 1, 1]\n", "Decoded MY: Euro 2020. Dia membuat penampilan sulung pada 2 Jun 2021 dalam perlawanan persahabatan menentang Wales, menggantikan Benjamin Pavard pada separuh masa pertama. Kehidupan\n", "Original JW: اييورو ٢٠٢٠. دي ممبوات ڤنمڤيلن سولوڠ ڤد ٢ جون ٢٠٢١ دالم ڤرلاونن ڤرساهابتن مننتڠ ولايس، مڠڬنتيكن بينجامين ڤاۏارد ڤد سيڤروح ماس ڤرتام. كهيدوڤن\n", "Tokenized JW: [8997, 3012, 63916, 310, 1607, 4955, 4845, 114, 140, 1780, 4190, 151, 2292, 12369, 2995, 8488, 63922, 4392, 27546, 56921, 16939, 114, 5163, 723, 664, 63916, 2820, 2, 64001, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded JW: اييورو ٢٠٢٠. دي ممبوات ڤنمڤيلن سولوڠ ڤد ٢ جون ٢٠٢١ دالم ڤرلاونن ڤرساهابتن مننتڠ ولايس، مڠڬنتيكن بينجامين ڤاۏارد ڤد سيڤروح ماس ڤرتام. كهيدوڤن\n", "-------------------------\n", "Original MY: Campora merupakan sebuah komun dan bandar yang terletak di Salerno di Campania dalam kawasan Itali.\n", "Tokenized MY: [8715, 6450, 221, 229, 712, 59, 879, 62, 299, 33, 49923, 33, 24635, 168, 469, 2187, 63916, 2, 64004, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded MY: Campora merupakan sebuah komun dan bandar yang terletak di Salerno di Campania dalam kawasan Itali.\n", "Original JW: چمڤور مروڤاكن سبواه كومون دان بندر يڠ ترلتق د ساليرنو د چمڤنيا دالم كاوسن إيطاليا.\n", "Tokenized JW: [12604, 220, 227, 765, 57, 714, 58, 298, 5, 49929, 5, 24626, 151, 452, 2086, 63916, 2, 64001, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded JW: چمڤور مروڤاكن سبواه كومون دان بندر يڠ ترلتق د ساليرنو د چمڤنيا دالم كاوسن إيطاليا.\n", "-------------------------\n", "Original MY: cara mengekalkan makanan (lazimnya ikan) dengan menggunakan garam (atau asam) yang banyak dan mampu untuk menghasilkan pekasam yang mampu bertahan lebih lama\n", "Tokenized MY: [3498, 5979, 2695, 156, 13, 5565, 171, 5078, 63953, 184, 1278, 12570, 156, 6915, 36662, 63953, 62, 1123, 59, 2761, 205, 2627, 3486, 42, 45, 62, 2761, 9154, 592, 2459, 2, 64004]\n", "Decoded MY: cara mengekalkan makanan (lazimnya ikan) dengan menggunakan garam (atau asam) yang banyak dan mampu untuk menghasilkan pekasam yang mampu bertahan lebih lama\n", "Original JW: چارا مڠكلكن ماكنن ﴿لازيمڽ ايكن﴾ دڠن مڠڬوناكن ڬارام ﴿اتاو اسم﴾ يڠ باڽق دان ممڤو اونتوق مڠحاصيلكن ڤيكاسم يڠ ممڤو برتاهن لبيه لاما\n", "Tokenized JW: [3077, 5953, 2404, 157, 61630, 63931, 4319, 63954, 176, 1262, 11228, 157, 6996, 18248, 63954, 58, 1048, 57, 2730, 203, 2615, 11490, 26720, 58, 2730, 9111, 582, 2012, 2, 64001, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded JW: چارا مڠكلكن ماكنن ﴿لازيمڽ ايكن﴾ دڠن مڠڬوناكن ڬارام ﴿اتاو اسم﴾ يڠ باڽق دان ممڤو اونتوق مڠحاصيلكن ڤيكاسم يڠ ممڤو برتاهن لبيه لاما\n", "-------------------------\n", "Original MY: dan TV12 ( TV12 Singapura ) di bawah nama STV12 ( Singapore Television Twelve ) mengendalikan pengurus Saluran 12 oleh Perdana 12 dan Premiere 12 dinamakan semula MediaCorp TV12\n", "Tokenized MY: [59, 3103, 8464, 156, 3103, 8464, 2763, 909, 33, 1039, 978, 54, 7242, 8464, 156, 18785, 21665, 9484, 88, 5269, 909, 8709, 9324, 16045, 2000, 277, 2992, 2000, 59, 18662, 2, 64004]\n", "Decoded MY: dan TV12 ( TV12 Singapura ) di bawah nama STV12 ( Singapore Television Twelve ) mengendalikan pengurus Saluran 12 oleh Perdana 12 dan Premi\n", "Original JW: دان تۏ١٢ ﴿ تۏ١٢ سيڠاڤورا ﴾ د باوه نام ستۏ١٢ ﴿ سيڠاڤوري تيليۏيسياون تويلۏي ﴾ مڠنداليكن ڤڠوروس سالورن ١٢ اوليه ڤردان ١٢ دان ڤريمياراي ١٢ ديناماكن سمولا ميدياچورڤ تۏ١٢\n", "Tokenized JW: [57, 2994, 3374, 157, 2994, 3374, 2754, 907, 5, 1020, 581, 716, 63948, 3374, 157, 18628, 19023, 59073, 378, 907, 8685, 7413, 4107, 1787, 270, 2340, 1787, 57, 11007, 2517, 1787, 3990, 1480, 40262, 2994, 3374, 2, 64001, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded JW: دان تۏ١٢ ﴿ تۏ١٢ سيڠاڤورا ﴾ د باوه نام ستۏ١٢ ﴿ سيڠاڤوري تيليۏيسياون تويلۏي ﴾ مڠنداليكن ڤڠوروس سالورن ١٢ اوليه ڤردان ١٢ دان ڤريمياراي ١٢ ديناماكن سمولا ميدياچورڤ تۏ١٢\n", "-------------------------\n", "Original MY: aras laut..\n", "Tokenized MY: [3472, 1401, 1168, 2, 64004, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded MY: aras laut..\n", "Original JW: ارس لاوت..\n", "Tokenized JW: [3325, 1058, 1168, 2, 64001, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", "Decoded JW: ارس لاوت..\n", "-------------------------\n", "dict_keys(['input_ids', 'attention_mask', 'labels'])\n" ] } ], "source": [ "# Test the data_collator\n", "debug_batch = [train_data[i] for i in range(5)] # Select a small batch for testing\n", "debug_inputs = data_collator(debug_batch)\n", "print(debug_inputs.keys())\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ptEw2_hGsHWv" }, "source": [ "**Inference**\n", "\n", "Let's perform the inference using the resulting model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nG4UOZ93sTrJ" }, "outputs": [], "source": [ "model = MBartForConditionalGeneration.from_pretrained(\"/content/drive/MyDrive/Education/Final Year Project I/Output3\")\n", "tokenizer = MBartTokenizer.from_pretrained(\"/content/drive/MyDrive/Education/Final Year Project I/Output3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cIamvBuysUcq", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "cdc68089-c5e1-42b3-fcc1-6b00044b229e" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Rumi - Boleh saya tau siapa nama awak?\n", "Jawi - بوليه ساي تاو سياڤا نام اوق\n" ] } ], "source": [ "sentence = \"Boleh saya tau siapa nama awak?\"\n", "inputs = tokenizer(sentence, return_tensors=\"pt\")\n", "translated_tokens = model.generate(**inputs, decoder_start_token_id=tokenizer.lang_code_to_id[\"ar_AR\"], early_stopping=True, max_length=48)\n", "pred = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]\n", "print(f\"Rumi - {sentence}\")\n", "print(f\"Jawi - {pred}\")" ] }, { "cell_type": "markdown", "source": [ "### **Fine Tuning the Model for Better Accuracy**" ], "metadata": { "id": "8yRlY21e2rVN" } }, { "cell_type": "markdown", "source": [ "Preparing the model and tokenizer" ], "metadata": { "id": "7MN21LE3ACl3" } }, { "cell_type": "code", "source": [ "model = MBartForConditionalGeneration.from_pretrained(\"/content/drive/MyDrive/Education/Final Year Project I/Output2\")\n", "tokenizer = MBartTokenizer.from_pretrained(\"/content/drive/MyDrive/Education/Final Year Project I/Output2\")" ], "metadata": { "id": "TY--Tu5_AFJo" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "Fine tuning" ], "metadata": { "id": "DQWG8MIsA4ZZ" } }, { "cell_type": "code", "source": [ "from transformers import (\n", " Seq2SeqTrainingArguments, Seq2SeqTrainer\n", ")\n", "import numpy as np\n", "import re\n", "\n", "result_dir = \"/content/drive/MyDrive/Education/Final Year Project I/Output3\"" ], "metadata": { "id": "KPy9C-TYA5iF" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "tokenizer.save_pretrained(result_dir)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "X5WcyFeVBXYy", "outputId": "6478a148-3155-4378-a791-3120bd6b843f" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "('/content/drive/MyDrive/Education/Final Year Project I/Output3/tokenizer_config.json',\n", " '/content/drive/MyDrive/Education/Final Year Project I/Output3/special_tokens_map.json',\n", " '/content/drive/MyDrive/Education/Final Year Project I/Output3/sentencepiece.bpe.model',\n", " '/content/drive/MyDrive/Education/Final Year Project I/Output3/added_tokens.json')" ] }, "metadata": {}, "execution_count": 18 } ] }, { "cell_type": "code", "source": [ "def data_collator(features: list):\n", " x = [f.get(\"translation\", {}).get(\"my\", \"\") for f in features]\n", " y = [f.get(\"translation\", {}).get(\"jw\", \"\") for f in features]\n", "\n", " inputs = tokenizer(x, return_tensors=\"pt\", padding='max_length', truncation=True, max_length=32)\n", "\n", " with tokenizer.as_target_tokenizer():\n", " inputs['labels'] = tokenizer(y, return_tensors=\"pt\", padding='max_length', truncation=True, max_length=48)['input_ids']\n", "\n", " return inputs" ], "metadata": { "id": "ljkyhM3bBi-m" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Converting dataset from list into Datasets format for Seq2Seq compatibility.\n", "\n", "from datasets import Dataset\n", "\n", "# Keep it random, Adam\n", "train_dataset = Dataset.from_list(train_part4) # Change the list based on the dataset part being used\n", "eval_dataset = Dataset.from_list(eval_part4) # Change the list based on the dataset part being used" ], "metadata": { "id": "9gJ-ywqkDYPl" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Hyperparameters\n", "batch_size = 4\n", "learning_rate = 3e-5\n", "epochs = 1" ], "metadata": { "id": "4Bitist2DcG-" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "args = Seq2SeqTrainingArguments(output_dir=result_dir,\n", " do_train=True,\n", " do_eval=True,\n", " per_device_train_batch_size=batch_size,\n", " per_device_eval_batch_size=batch_size,\n", " learning_rate=learning_rate,\n", " num_train_epochs=epochs,\n", " eval_strategy=\"epoch\",\n", " remove_unused_columns=False,\n", " save_strategy=\"steps\",\n", " save_steps=10000,\n", " save_total_limit=2,\n", " report_to=\"none\",\n", " )\n", "\n", "trainer = Seq2SeqTrainer(model=model,\n", " args=args,\n", " data_collator=data_collator,\n", " train_dataset=train_dataset,\n", " eval_dataset=eval_dataset,\n", " )" ], "metadata": { "id": "n0dJz_m3DeGC" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "trainer.train()\n", "trainer.save_model(result_dir)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 358 }, "id": "xbNXhLygDnJ3", "outputId": "2bf6a254-c6a9-49e2-cd06-a4e040e382bd" }, "execution_count": null, "outputs": [ { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3953: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [16304/37490 30:18 < 39:22, 8.97 it/s, Epoch 0.43/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3953: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n", " warnings.warn(\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "" ], "text/html": [ "\n", "

\n", " \n", " \n", " [37490/37490 1:12:38, Epoch 1/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
11.3704001.284363

" ] }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3953: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3953: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:3953: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n", " warnings.warn(\n" ] } ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "00e24c6f5f6b462387c4f41668145a6b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_c30dea49fc6d46388b028c2ae3b2078a", "IPY_MODEL_2bc72303aba44d4c8fcd2fe25714b26b", "IPY_MODEL_1f75d00957a6424686725b269c29c3f5" ], "layout": "IPY_MODEL_26c1d1577ee243068a9e92e3d2ed7ef1" } }, "04018a70f1ff4d3da242878e745aea66": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "04841423621f4c6ba9d257210a906ad5": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0a012e2534d7482581c8667ba5497355": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0f1787c411254ad1b37fbb155684c942": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1595156d2bf541afbce127a3ad7eb986": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "192ba7d635894855bd3463bdda35e0a3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1f75d00957a6424686725b269c29c3f5": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_04018a70f1ff4d3da242878e745aea66", "placeholder": "​", "style": "IPY_MODEL_4b31f4aff6e941d18d5ebc55870d6951", "value": " 749797/749797 [00:03<00:00, 197377.37 examples/s]" } }, "22baeaa9f7fb4eceb9a40929d7ee07f5": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0f1787c411254ad1b37fbb155684c942", "placeholder": "​", "style": "IPY_MODEL_d0bcebb2cfa241d6aaebd9ab1c4a412f", "value": "Map: 100%" } }, "268a7b7d7bc44775aad99e277ce7c105": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "26c1d1577ee243068a9e92e3d2ed7ef1": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2ab55ed1aec549abb6432bd3775f6a59": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a0bf9ddb98314d42873111a51eff40b0", "max": 749797, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_f8d5799e862e4c3d872b3955737b0023", "value": 749797 } }, "2bc72303aba44d4c8fcd2fe25714b26b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1595156d2bf541afbce127a3ad7eb986", "max": 749797, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_e92fbb2bad5048ccb18d275bc852771c", "value": 749797 } }, "329046c64bf34965abce9e9354d5bd0d": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d9ab8a0801f74d35b749008f59cfe81a", "placeholder": "​", "style": "IPY_MODEL_98516005540d42d68a9a5539c4c67e40", "value": "Saving the dataset (2/2 shards): 100%" } }, "3ead78b402d44bc1afa74b76c05b4c70": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "429c06890a784aceb0623402b6fba0c9": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "42f918171a8c4e66a59252c6dffb8a6a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cccdef69b1c04df1870d868a711595b1", "placeholder": "​", "style": "IPY_MODEL_5400ca171013472f8b8d6c15c3e86589", "value": " 1499594/1499594 [00:08<00:00, 165063.36 examples/s]" } }, "496f3711fc934d999c55486e6cd42ca2": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "4b31f4aff6e941d18d5ebc55870d6951": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "52be4e574ef449ad85235b5bce17dcdd": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "5400ca171013472f8b8d6c15c3e86589": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5f6312c9836d4393a28ee379bd0e5a66": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5fbc47165db74f91b9e614df0d7df7fd": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e4edb8c36b374e20bf9531985a86b96b", "IPY_MODEL_988b0139b0d8499da97cb7d3a5607ce4", "IPY_MODEL_42f918171a8c4e66a59252c6dffb8a6a" ], "layout": "IPY_MODEL_901560f4d7b7467288baa07c12c62f04" } }, "61077ce93ce54a6291ae4c1e3f0188bd": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f0e7cb46d1a44e6ba127430311e663e3", "placeholder": "​", "style": "IPY_MODEL_192ba7d635894855bd3463bdda35e0a3", "value": " 1499594/1499594 [00:08<00:00, 156381.23 examples/s]" } }, "61d50a37307e43dbb1ac181125bab487": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "630efe0695414f7f8528fdcfdd41760e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a00305d6a2d14fdf842b081caf6d5069", "placeholder": "​", "style": "IPY_MODEL_a93e50692ba144648286789b1ba9cc64", "value": " 749797/749797 [00:30<00:00, 17648.54 examples/s]" } }, "6417468ac85b4f52b53b67ca9f7fe6e1": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0a012e2534d7482581c8667ba5497355", "max": 1499594, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_52be4e574ef449ad85235b5bce17dcdd", "value": 1499594 } }, "65a7af49974d4745b92862c026738aa6": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_22baeaa9f7fb4eceb9a40929d7ee07f5", "IPY_MODEL_2ab55ed1aec549abb6432bd3775f6a59", "IPY_MODEL_630efe0695414f7f8528fdcfdd41760e" ], "layout": "IPY_MODEL_da89107c88944b4cb996ce2122bda4ca" } }, "6a7ffce606a8400fa456ca2c22d834f1": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_329046c64bf34965abce9e9354d5bd0d", "IPY_MODEL_9bf19286853148c4bcd2bec31e5505d0", "IPY_MODEL_a09cddb7290b4c2bb794cbf16b03e6ad" ], "layout": "IPY_MODEL_61d50a37307e43dbb1ac181125bab487" } }, "84c4b78940d24c9e82c24edfdf01a5fc": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "855af7723a93453e9bd9eef111ff8b34": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "901560f4d7b7467288baa07c12c62f04": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "94e1781694124a8ba77539a435d062de": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_04841423621f4c6ba9d257210a906ad5", "placeholder": "​", "style": "IPY_MODEL_9944b8281bfd494ebf67cc72ff8f27a8", "value": "Filter: 100%" } }, "98516005540d42d68a9a5539c4c67e40": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "988b0139b0d8499da97cb7d3a5607ce4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_daefc42c94904556806e88b19edbcc77", "max": 1499594, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_855af7723a93453e9bd9eef111ff8b34", "value": 1499594 } }, "9944b8281bfd494ebf67cc72ff8f27a8": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9a8a0fc6c55c473e8ab7eb7db9dc34b4": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9bf19286853148c4bcd2bec31e5505d0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_268a7b7d7bc44775aad99e277ce7c105", "max": 749797, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ebf425f5e3a24ce299e6bfdbfc4a0810", "value": 749797 } }, "a00305d6a2d14fdf842b081caf6d5069": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a09cddb7290b4c2bb794cbf16b03e6ad": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5f6312c9836d4393a28ee379bd0e5a66", "placeholder": "​", "style": "IPY_MODEL_3ead78b402d44bc1afa74b76c05b4c70", "value": " 749797/749797 [00:03<00:00, 206299.91 examples/s]" } }, "a0bf9ddb98314d42873111a51eff40b0": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a93e50692ba144648286789b1ba9cc64": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "aec7f152f8b74666a49a04e643244e1c": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b1d13ab11f664bd2a76d5f2cf1b17cff": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_94e1781694124a8ba77539a435d062de", "IPY_MODEL_6417468ac85b4f52b53b67ca9f7fe6e1", "IPY_MODEL_61077ce93ce54a6291ae4c1e3f0188bd" ], "layout": "IPY_MODEL_84c4b78940d24c9e82c24edfdf01a5fc" } }, "c30dea49fc6d46388b028c2ae3b2078a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9a8a0fc6c55c473e8ab7eb7db9dc34b4", "placeholder": "​", "style": "IPY_MODEL_496f3711fc934d999c55486e6cd42ca2", "value": "Saving the dataset (2/2 shards): 100%" } }, "cccdef69b1c04df1870d868a711595b1": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d0bcebb2cfa241d6aaebd9ab1c4a412f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d9ab8a0801f74d35b749008f59cfe81a": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "da89107c88944b4cb996ce2122bda4ca": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "daefc42c94904556806e88b19edbcc77": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e4edb8c36b374e20bf9531985a86b96b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_aec7f152f8b74666a49a04e643244e1c", "placeholder": "​", "style": "IPY_MODEL_429c06890a784aceb0623402b6fba0c9", "value": "Filter: 100%" } }, "e92fbb2bad5048ccb18d275bc852771c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ebf425f5e3a24ce299e6bfdbfc4a0810": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "f0e7cb46d1a44e6ba127430311e663e3": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f8d5799e862e4c3d872b3955737b0023": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "b44a23b1c635457f90cfcfacadfeaee8": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_cf5126785d6d4fa99e3cefa410beea30", "IPY_MODEL_0f28f16a797d4871b475db465a7dfe3c", "IPY_MODEL_c72f82181b5c4d7495d8f1e0da173004" ], "layout": "IPY_MODEL_7d65829cc25945cc956aee274f7c95cd" } }, "cf5126785d6d4fa99e3cefa410beea30": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_263a8fd4ba4941c5aba79a6a14872a18", "placeholder": "​", "style": "IPY_MODEL_f3777d69e44b4957a79c2348d471ca5e", "value": "config.json: 100%" } }, "0f28f16a797d4871b475db465a7dfe3c": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a514eb8ea31c4f46b948f18411221735", "max": 1185, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_c2ce61060d7a49afb5015b30845b61f8", "value": 1185 } }, "c72f82181b5c4d7495d8f1e0da173004": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_255e8806c47d4abab7aa41e393ba18e8", "placeholder": "​", "style": "IPY_MODEL_2d317a350d7341f99175034ec4c52dcb", "value": " 1.19k/1.19k [00:00<00:00, 86.2kB/s]" } }, "7d65829cc25945cc956aee274f7c95cd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "263a8fd4ba4941c5aba79a6a14872a18": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f3777d69e44b4957a79c2348d471ca5e": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "a514eb8ea31c4f46b948f18411221735": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c2ce61060d7a49afb5015b30845b61f8": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "255e8806c47d4abab7aa41e393ba18e8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2d317a350d7341f99175034ec4c52dcb": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e2bad09a6029485193f85a986c014c4c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_c433f05101694e3881d44b5a597077f2", "IPY_MODEL_625c961e0a49419eac19edf2d7f70e5d", "IPY_MODEL_0f4000463e064fcdb2f3c6023c4bbf74" ], "layout": "IPY_MODEL_8db25c530b4c4f6185f9844dd7610c1e" } }, "c433f05101694e3881d44b5a597077f2": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c6e70930a4f44fee9f9baaf86bef2692", "placeholder": "​", "style": "IPY_MODEL_962ea77573d94458b6b02192d2415bda", "value": "pytorch_model.bin: 100%" } }, "625c961e0a49419eac19edf2d7f70e5d": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e5137a0d7d7a493eb22d9b3cdea4cb3f", "max": 2444517405, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_72c164cc14584029a7fca1f3c22f7e84", "value": 2444517405 } }, "0f4000463e064fcdb2f3c6023c4bbf74": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_05a7b2ada59c48c6b9774cc725fc854f", "placeholder": "​", "style": "IPY_MODEL_6985a0c76d64485e8b8aa7c0a9383690", "value": " 2.44G/2.44G [01:42<00:00, 24.3MB/s]" } }, "8db25c530b4c4f6185f9844dd7610c1e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c6e70930a4f44fee9f9baaf86bef2692": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "962ea77573d94458b6b02192d2415bda": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e5137a0d7d7a493eb22d9b3cdea4cb3f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "72c164cc14584029a7fca1f3c22f7e84": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "05a7b2ada59c48c6b9774cc725fc854f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6985a0c76d64485e8b8aa7c0a9383690": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "dce93797096f4aaca2cec4a464150027": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_77b7f243f8d543f7816da869ef5997e9", "IPY_MODEL_29c3cf060bf64fb281346c9782b52b20", "IPY_MODEL_889d0460eb82410e86fa90dd28c79a0b" ], "layout": "IPY_MODEL_fd723c6c74f0412896fc9b63f359f799" } }, "77b7f243f8d543f7816da869ef5997e9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_385552daaa104ba2b2c8966945a556f4", "placeholder": "​", "style": "IPY_MODEL_69648088f19345b3973330b733aa3362", "value": "generation_config.json: 100%" } }, "29c3cf060bf64fb281346c9782b52b20": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_45b237badb6b4368ae03a64b33c7ab45", "max": 205, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_0f642650516f4e34a4371dd7d6c8ed23", "value": 205 } }, "889d0460eb82410e86fa90dd28c79a0b": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ab126ab44440479c88d638799e343ab2", "placeholder": "​", "style": "IPY_MODEL_32ba466b02c14bdf90f97b14a587cafa", "value": " 205/205 [00:00<00:00, 14.1kB/s]" } }, "fd723c6c74f0412896fc9b63f359f799": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "385552daaa104ba2b2c8966945a556f4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "69648088f19345b3973330b733aa3362": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "45b237badb6b4368ae03a64b33c7ab45": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0f642650516f4e34a4371dd7d6c8ed23": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ab126ab44440479c88d638799e343ab2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "32ba466b02c14bdf90f97b14a587cafa": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "ec7d5ea7dedf4cb48d1262620a3790bb": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5f731ca5ce7a49aca73fd0d1de61c644", "IPY_MODEL_f95150ea42c648e68eeb5295cdefe1cd", "IPY_MODEL_095375d20ee94ae586327ed987c94b47" ], "layout": "IPY_MODEL_5367fd052adc49da817a9b5519da5778" } }, "5f731ca5ce7a49aca73fd0d1de61c644": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_36f2a8ae25424c31a702b1e5b2f548ec", "placeholder": "​", "style": "IPY_MODEL_1c62938e7f39440ea47483be4181bb33", "value": "sentencepiece.bpe.model: 100%" } }, "f95150ea42c648e68eeb5295cdefe1cd": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b1bb09fc8023467292ecbc307f7b4c94", "max": 5069051, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_33d23b0dec2f45f2adae2707ad8138ca", "value": 5069051 } }, "095375d20ee94ae586327ed987c94b47": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_df0a74a9529c40e7a20b5e64af9c0cc8", "placeholder": "​", "style": "IPY_MODEL_8433150bd5d146229ffc614565dce32e", "value": " 5.07M/5.07M [00:00<00:00, 23.9MB/s]" } }, "5367fd052adc49da817a9b5519da5778": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "36f2a8ae25424c31a702b1e5b2f548ec": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1c62938e7f39440ea47483be4181bb33": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b1bb09fc8023467292ecbc307f7b4c94": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "33d23b0dec2f45f2adae2707ad8138ca": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "df0a74a9529c40e7a20b5e64af9c0cc8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8433150bd5d146229ffc614565dce32e": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "84ff9010c0274ad9a28ff85284bd9a77": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_d3dd4a2187424f3381a90dec06aabd9f", "IPY_MODEL_ee7ed503ce844ee9b043af1160a5ddfe", "IPY_MODEL_72a8d36af5eb4243803e987a8b596d6c" ], "layout": "IPY_MODEL_d788f100ecd643b8bb98eb2d7c8dd59f" } }, "d3dd4a2187424f3381a90dec06aabd9f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_89efc9c3257142d78cc560833cb4c203", "placeholder": "​", "style": "IPY_MODEL_fd47aa11cfac4921902ac80fe6702342", "value": "tokenizer.json: 100%" } }, "ee7ed503ce844ee9b043af1160a5ddfe": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_68d8c441e87f40829ff1f5e5e89e6e82", "max": 9099897, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_dc350db4006542e5aa101514842a00bf", "value": 9099897 } }, "72a8d36af5eb4243803e987a8b596d6c": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_23589ab7e0284eec9f48487c8bb3bd70", "placeholder": "​", "style": "IPY_MODEL_b4c291d5efa8429c8821dc87974bd701", "value": " 9.10M/9.10M [00:00<00:00, 16.5MB/s]" } }, "d788f100ecd643b8bb98eb2d7c8dd59f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "89efc9c3257142d78cc560833cb4c203": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fd47aa11cfac4921902ac80fe6702342": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "68d8c441e87f40829ff1f5e5e89e6e82": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dc350db4006542e5aa101514842a00bf": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "23589ab7e0284eec9f48487c8bb3bd70": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b4c291d5efa8429c8821dc87974bd701": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "nbformat": 4, "nbformat_minor": 0 }