ArthurZ HF staff commited on
Commit
05d7cc0
1 Parent(s): 8f57f08

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -21,6 +21,7 @@
21
  "legacy": false,
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
 
24
  "sp_model_kwargs": {},
25
  "tokenizer_class": "LlamaTokenizer",
26
  "unk_token": {
@@ -30,6 +31,5 @@
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false
33
- },
34
- "use_fast": false
35
  }
 
21
  "legacy": false,
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
24
+ "padding_side": "right",
25
  "sp_model_kwargs": {},
26
  "tokenizer_class": "LlamaTokenizer",
27
  "unk_token": {
 
31
  "normalized": false,
32
  "rstrip": false,
33
  "single_word": false
34
+ }
 
35
  }