Upload tokenizer

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 5,
+      "id": 3,
       "content": "UTT_BOUNDARY",
       "single_word": false,
       "lstrip": false,
@@ -34,13 +34,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "\n"
-        },
-        "content": " UTT_BOUNDARY"
-      },
       {
         "type": "Strip",
         "strip_left": true,
@@ -51,42 +44,93 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
   "decoder": null,
   "model": {
     "type": "WordLevel",
     "vocab": {
       "UNK": 0,
       "PAD": 1,
-      "BOS": 2,
-      "EOS": 3,
-      "WORD_BOUNDARY": 4,
-      "UTT_BOUNDARY": 5,
-      "k": 6,
-      "s": 7,
-      "o": 8,
-      "b": 9,
-      "a": 10,
-      "h": 11,
-      "n": 12,
-      "t̠ʃ": 13,
-      "i": 14,
-      "j": 15,
-      "d": 16,
-      "e": 17,
-      "ʃ": 18,
-      "u": 19,
-      "ɡ": 20,
-      "r": 21,
-      "f": 22,
-      "t": 23,
-      "m": 24,
-      "d̠ʒ": 25,
-      "l": 26,
-      "q": 27,
-      "v": 28,
-      "z": 29,
-      "p": 30
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "k": 4,
+      "s": 5,
+      "o": 6,
+      "b": 7,
+      "a": 8,
+      "h": 9,
+      "n": 10,
+      "t̠ʃ": 11,
+      "i": 12,
+      "j": 13,
+      "d": 14,
+      "e": 15,
+      "ʃ": 16,
+      "u": 17,
+      "ɡ": 18,
+      "r": 19,
+      "f": 20,
+      "t": 21,
+      "m": 22,
+      "d̠ʒ": 23,
+      "l": 24,
+      "q": 25,
+      "v": 26,
+      "z": 27,
+      "p": 28
     },
     "unk_token": "UNK"
   }

tokenizer_config.json

@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "3": {
       "content": "UTT_BOUNDARY",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"k":6,"s":7,"o":8,"b":9,"a":10,"h":11,"n":12,"t̠ʃ":13,"i":14,"j":15,"d":16,"e":17,"ʃ":18,"u":19,"ɡ":20,"r":21,"f":22,"t":23,"m":24,"d̠ʒ":25,"l":26,"q":27,"v":28,"z":29,"p":30}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"k":4,"s":5,"o":6,"b":7,"a":8,"h":9,"n":10,"t̠ʃ":11,"i":12,"j":13,"d":14,"e":15,"ʃ":16,"u":17,"ɡ":18,"r":19,"f":20,"t":21,"m":22,"d̠ʒ":23,"l":24,"q":25,"v":26,"z":27,"p":28}