floriangardin commited on
Commit
d006aae
·
1 Parent(s): 0b68287

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -9,7 +9,7 @@
9
  },
10
  "post_processor": null,
11
  "decoder": {
12
- "type": "Split"
13
  },
14
  "model": {
15
  "type": "WordLevel",
 
9
  },
10
  "post_processor": null,
11
  "decoder": {
12
+ "type": "Strip"
13
  },
14
  "model": {
15
  "type": "WordLevel",