Hugging Face's logo

alehc
/
swissai-tokenizer

Transformers
Inference Endpoints
Model card Files Community
alehc commited on
Commit
781b138
·
verified ·
1 Parent(s): 2958d0f

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -1
  2. tokenizer_config.json +3 -3
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f61c43a51782aacaf4eaadb53f1c446019269f0de2caddd55408cba00b76ece
3
  size 17078368
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be12f4375d655cc740864e3a9041bcddd8477942f209d9e7f27f6c8767162638
3
  size 17078368
tokenizer_config.json CHANGED
@@ -468,7 +468,7 @@
468
  "special": true
469
  },
470
  "58": {
471
- "content": "<SPECIAL_58>",
472
  "lstrip": false,
473
  "normalized": false,
474
  "rstrip": false,
@@ -476,7 +476,7 @@
476
  "special": true
477
  },
478
  "59": {
479
- "content": "<SPECIAL_59>",
480
  "lstrip": false,
481
  "normalized": false,
482
  "rstrip": false,
@@ -484,7 +484,7 @@
484
  "special": true
485
  },
486
  "60": {
487
- "content": "<SPECIAL_60>",
488
  "lstrip": false,
489
  "normalized": false,
490
  "rstrip": false,
 
468
  "special": true
469
  },
470
  "58": {
471
+ "content": "<|fim_begin|>",
472
  "lstrip": false,
473
  "normalized": false,
474
  "rstrip": false,
 
476
  "special": true
477
  },
478
  "59": {
479
+ "content": "<|fim_hole|>",
480
  "lstrip": false,
481
  "normalized": false,
482
  "rstrip": false,
 
484
  "special": true
485
  },
486
  "60": {
487
+ "content": "<|fim_end|>",
488
  "lstrip": false,
489
  "normalized": false,
490
  "rstrip": false,