Raghavan commited on
Commit
4054959
1 Parent(s): 51d780a

Upload 6 files

Browse files
sentencepiece.bpe.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
3
- size 5069051
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f5e2fefcf793761a76a6bfb8ad35489f9c203b25557673284b6d032f41043f4
3
+ size 1356293
special_tokens_map.json CHANGED
@@ -7,7 +7,8 @@
7
  "lstrip": true,
8
  "normalized": true,
9
  "rstrip": false,
10
- "single_word": false
 
11
  },
12
  "pad_token": "<pad>",
13
  "sep_token": "</s>",
 
7
  "lstrip": true,
8
  "normalized": true,
9
  "rstrip": false,
10
+ "single_word": false,
11
+ "special": false
12
  },
13
  "pad_token": "<pad>",
14
  "sep_token": "</s>",
tokenizer_config.json CHANGED
@@ -9,7 +9,8 @@
9
  "lstrip": true,
10
  "normalized": true,
11
  "rstrip": false,
12
- "single_word": false
 
13
  },
14
  "model_max_length": 1000000000000000019884624838656,
15
  "pad_token": "<pad>",
 
9
  "lstrip": true,
10
  "normalized": true,
11
  "rstrip": false,
12
+ "single_word": false,
13
+ "special": false
14
  },
15
  "model_max_length": 1000000000000000019884624838656,
16
  "pad_token": "<pad>",