floriangardin
commited on
Commit
·
d006aae
1
Parent(s):
0b68287
Update tokenizer.json
Browse files- tokenizer.json +1 -1
tokenizer.json
CHANGED
@@ -9,7 +9,7 @@
|
|
9 |
},
|
10 |
"post_processor": null,
|
11 |
"decoder": {
|
12 |
-
"type": "
|
13 |
},
|
14 |
"model": {
|
15 |
"type": "WordLevel",
|
|
|
9 |
},
|
10 |
"post_processor": null,
|
11 |
"decoder": {
|
12 |
+
"type": "Strip"
|
13 |
},
|
14 |
"model": {
|
15 |
"type": "WordLevel",
|