Upload tokenizer

special_tokens_map.json

@@ -7,7 +7,7 @@
     "single_word": false
   },
   "cls_token": {
-    "content": "<CLS|LLM-jp>",
+    "content": "<s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
@@ -21,21 +21,21 @@
     "single_word": false
   },
   "mask_token": {
-    "content": "<MASK|LLM-jp>",
+    "content": "<MASK>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<PAD|LLM-jp>",
+    "content": "<PAD>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "sep_token": {
-    "content": "<SEP|LLM-jp>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "<MASK|LLM-jp>",
+      "content": "<MASK>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 4,
-      "content": "<PAD|LLM-jp>",
+      "content": "<PAD>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,7 +50,7 @@
     },
     {
       "id": 5,
-      "content": "<CLS|LLM-jp>",
+      "content": "<CLS>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 6,
-      "content": "<SEP|LLM-jp>",
+      "content": "<SEP>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
     },
     {
       "id": 7,
-      "content": "<EOD|LLM-jp>",
+      "content": "<EOD>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -110,6 +110,12 @@
           "id": "A",
           "type_id": 0
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
       }
     ],
     "pair": [
@@ -127,8 +133,8 @@
       },
       {
         "SpecialToken": {
-          "id": "<s>",
-          "type_id": 1
+          "id": "</s>",
+          "type_id": 0
         }
       },
       {
@@ -136,9 +142,24 @@
           "id": "B",
           "type_id": 1
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 1
+        }
       }
     ],
     "special_tokens": {
+      "</s>": {
+        "id": "</s>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "</s>"
+        ]
+      },
       "<s>": {
         "id": "<s>",
         "ids": [
@@ -192,23 +213,23 @@
         -127.5
       ],
       [
-        "<MASK|LLM-jp>",
+        "<MASK>",
         -127.5
       ],
       [
-        "<PAD|LLM-jp>",
+        "<PAD>",
         -127.5
       ],
       [
-        "<CLS|LLM-jp>",
+        "<CLS>",
         -127.5
       ],
       [
-        "<SEP|LLM-jp>",
+        "<SEP>",
         -127.5
       ],
       [
-        "<EOD|LLM-jp>",
+        "<EOD>",
         -127.5
       ],
       [

tokenizer_config.json

@@ -1,6 +1,4 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -27,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "<MASK|LLM-jp>",
+      "content": "<MASK>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -35,7 +33,7 @@
       "special": true
     },
     "4": {
-      "content": "<PAD|LLM-jp>",
+      "content": "<PAD>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -43,7 +41,7 @@
       "special": true
     },
     "5": {
-      "content": "<CLS|LLM-jp>",
+      "content": "<CLS>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -51,7 +49,7 @@
       "special": true
     },
     "6": {
-      "content": "<SEP|LLM-jp>",
+      "content": "<SEP>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -59,7 +57,7 @@
       "special": true
     },
     "7": {
-      "content": "<EOD|LLM-jp>",
+      "content": "<EOD>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -69,14 +67,14 @@
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "cls_token": "<CLS|LLM-jp>",
-  "eod_token": "</s>",
+  "cls_token": "<s>",
+  "eod_token": "<EOD>",
   "eos_token": "</s>",
   "extra_ids": 0,
-  "mask_token": "<MASK|LLM-jp>",
+  "mask_token": "<MASK>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<PAD|LLM-jp>",
-  "sep_token": "<SEP|LLM-jp>",
+  "pad_token": "<PAD>",
+  "sep_token": "</s>",
   "sp_model_kwargs": {},
   "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "<unk>"