Xenova
/

esm2_t6_8M_UR50D

Transformers.js

Model card Files Files and versions Community

Xenova HF staff commited on Jan 4, 2024

Commit

ed9f336

·

1 Parent(s): dab9672

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +18 -0

tokenizer.json CHANGED Viewed

@@ -328,17 +328,35 @@
       }
     ],
     "pair": [
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
       }
     ],
     "special_tokens": {

       }
     ],
     "pair": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<eos>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<eos>",
+          "type_id": 1
+        }
       }
     ],
     "special_tokens": {