data/lang_bpe_500/tokens.txt

<blk> 0
<sos/eos> 1
<unk> 2
S 3
▁THE 4
' 5
T 6
▁I 7
▁IT 8
E 9
▁YEAH 10
▁A 11
▁UH 12
▁YOU 13
▁THAT 14
▁TO 15
▁WE 16
▁AND 17
▁SO 18
A 19
D 20
I 21
▁OF 22
P 23
ING 24
M 25
O 26
▁UM 27
U 28
▁IN 29
▁IS 30
C 31
N 32
▁HAVE 33
▁S 34
RE 35
▁OKAY 36
▁BE 37
▁BUT 38
ER 39
L 40
▁FOR 41
▁MM 42
▁LIKE 43
Y 44
▁CAN 45
▁THIS 46
▁THINK 47
▁ON 48
▁P 49
▁C 50
▁JUST 51
R 52
▁IF 53
▁F 54
ED 55
IN 56
▁WHAT 57
▁DO 58
▁OR 59
AR 60
▁ 61
▁T 62
G 63
▁WELL 64
▁NO 65
▁ONE 66
▁NOT 67
F 68
▁WITH 69
▁L 70
▁KNOW 71
AL 72
▁ARE 73
▁M 74
- 75
▁W 76
H 77
▁THEY 78
LL 79
▁DON 80
B 81
▁THEN 82
HMM 83
K 84
VE 85
▁RE 86
LE 87
EN 88
▁THERE 89
▁D 90
OR 91
W 92
▁WOULD 93
▁RIGHT 94
▁REMOTE 95
AN 96
LY 97
▁SOME 98
ON 99
▁USE 100
▁E 101
IT 102
▁WAS 103
▁OH 104
▁ABOUT 105
▁MEAN 106
ION 107
▁ALL 108
▁AS 109
▁B 110
IC 111
▁SOMETHING 112
IS 113
▁MAYBE 114
▁CONTROL 115
▁BECAUSE 116
ES 117
▁SHOULD 118
RA 119
▁AT 120
▁GO 121
▁WANT 122
▁COULD 123
▁AN 124
▁LOOK 125
▁MORE 126
▁V 127
▁R 128
▁SEE 129
CH 130
▁WILL 131
▁MAKE 132
▁WHICH 133
▁SAY 134
▁MA 135
▁YOUR 136
▁TH 137
V 138
▁THING 139
▁HOW 140
▁TWO 141
ENT 142
▁GET 143
▁YES 144
▁NEED 145
▁VERY 146
▁ST 147
▁' 148
ATE 149
▁G 150
▁UP 151
▁REALLY 152
▁BUTTONS 153
▁KIND 154
▁OUR 155
▁N 156
▁GONNA 157
▁FROM 158
▁ALSO 159
US 160
▁GOOD 161
▁DESIGN 162
▁PEOPLE 163
UN 164
RO 165
▁WORK 166
ATION 167
▁HE 168
▁O 169
▁HERE 170
▁COM 171
▁MO 172
▁OUT 173
▁CO 174
UR 175
▁TIME 176
CK 177
▁WHEN 178
▁H 179
IL 180
AT 181
▁US 182
▁GOT 183
▁NOW 184
▁OTHER 185
MENT 186
▁ME 187
▁DE 188
▁MY 189
▁BUTTON 190
VER 191
▁PUT 192
ABLE 193
▁LO 194
▁DOES 195
ID 196
▁FUNCTION 197
▁THINGS 198
X 199
▁EX 200
▁THAN 201
▁SORT 202
▁CON 203
▁HMM 204
LA 205
▁Y 206
CAUSE 207
AGE 208
LO 209
▁ANY 210
▁DI 211
▁COLOUR 212
▁DIFFERENT 213
▁IDEA 214
▁MEETING 215
▁ACTUALLY 216
IGHT 217
▁THESE 218
▁HAS 219
▁BIT 220
▁CH 221
▁POINT 222
▁SH 223
TH 224
▁WHERE 225
GE 226
▁THEM 227
▁MUCH 228
▁HO 229
▁FIVE 230
▁SURE 231
▁TOO 232
▁CHANNEL 233
▁ONLY 234
▁GOING 235
▁WAY 236
▁USER 237
▁THREE 238
▁LITTLE 239
▁NEW 240
Z 241
▁YEP 242
▁MIGHT 243
▁BY 244
▁GIVE 245
▁QUITE 246
▁PROBABLY 247
IVE 248
▁TAKE 249
▁LET 250
UM 251
▁FIRST 252
ALLY 253
▁INTO 254
▁SCREEN 255
▁HAD 256
▁BACK 257
▁BA 258
▁EVEN 259
▁PRO 260
▁MARKET 261
▁LOT 262
▁STILL 263
▁THOUGH 264
ITY 265
▁THOSE 266
▁ALRIGHT 267
▁SAME 268
▁FIND 269
▁DID 270
▁MOST 271
▁GUESS 272
▁TALK 273
▁DOWN 274
▁OVER 275
▁UN 276
▁PRODUCT 277
▁STUFF 278
▁NEXT 279
▁PART 280
▁HAND 281
▁PROBLEM 282
▁START 283
IES 284
▁RECOGNITION 285
▁J 286
▁COST 287
▁FOUR 288
▁TRY 289
▁EASY 290
▁BASICALLY 291
▁AGAIN 292
▁SPEECH 293
HUH 294
▁COME 295
▁RUBBER 296
▁BIG 297
▁INTER 298
▁IMPORTANT 299
▁NUMBER 300
▁PROJECT 301
▁BR 302
▁SIMPLE 303
▁CHIP 304
▁WORD 305
KAY 306
▁SCROLL 307
▁CHANGE 308
▁CURVE 309
▁WHO 310
▁INTERFACE 311
▁CASE 312
FUL 313
▁SORRY 314
▁SET 315
▁FEATURE 316
▁SHAPE 317
▁WHY 318
▁SAID 319
▁TRUE 320
▁POWER 321
▁INFORMATION 322
▁OPTION 323
▁DIDN 324
▁ADD 325
▁EACH 326
▁THEIR 327
ANCE 328
▁DISPLAY 329
▁FEEL 330
▁KEEP 331
▁K 332
▁SIX 333
▁DISCUSS 334
▁EXAMPLE 335
▁QUESTION 336
▁COURSE 337
▁ALREADY 338
▁TELEVISION 339
▁WANNA 340
▁LINE 341
▁ANYTHING 342
▁VOLUME 343
▁MENU 344
▁NICE 345
▁EVERY 346
▁PRESENTATION 347
▁THROUGH 348
▁BETTER 349
▁WHATEVER 350
ENCE 351
▁WHOLE 352
▁EXACTLY 353
▁EVERYTHING 354
▁ANOTHER 355
▁VOICE 356
▁LAST 357
▁PUSH 358
▁HAVING 359
▁ANYWAY 360
▁DEVICE 361
▁DATA 362
▁AFTER 363
▁SPEAK 364
▁BEFORE 365
▁DRAW 366
▁EURO 367
▁COOL 368
▁DECIDE 369
▁SEVEN 370
▁PRESS 371
▁PRETTY 372
▁TREND 373
▁FRUIT 374
▁LONG 375
▁TWENTY 376
▁BASIC 377
▁SUPPOSE 378
▁POSSIBLE 379
▁SMALL 380
▁REQUIRE 381
▁MANY 382
▁DUNNO 383
▁HIGH 384
▁ELSE 385
▁SIDE 386
▁BETWEEN 387
▁SPECIAL 388
▁BATTERY 389
▁PLASTIC 390
▁HARD 391
▁SLIDE 392
▁WHEEL 393
▁SOUND 394
▁MATERIAL 395
▁SECOND 396
▁ENOUGH 397
▁PLAY 398
▁WHETHER 399
▁SEGMENT 400
▁READ 401
▁BOTH 402
▁ALWAYS 403
▁COMPONENT 404
▁FILE 405
▁PROGRAM 406
▁LESS 407
▁TELL 408
▁SIGNAL 409
▁FANCY 410
▁SHOW 411
▁MADE 412
▁YELLOW 413
▁PERCENT 414
▁GREAT 415
▁TURN 416
▁AGREE 417
▁FASHION 418
▁FIFTY 419
▁TOGETHER 420
▁FACT 421
▁AROUND 422
▁WATCH 423
▁EXPENSIVE 424
▁TWELVE 425
▁TYPE 426
▁LEFT 427
▁EXTRA 428
▁NAME 429
▁DEFINITELY 430
▁CONCEPT 431
▁ENERGY 432
▁BOARD 433
▁OPEN 434
▁LEAST 435
J 436
▁THIRTY 437
▁GROUP 438
▁KEY 439
▁SPECIFIC 440
▁INTERESTING 441
▁PRICE 442
▁UNDER 443
▁PHONE 444
▁TECHNOLOGY 445
▁TELETEXT 446
▁SENSE 447
▁GUYS 448
▁BANANA 449
▁DOUBLE 450
▁HALF 451
▁SYSTEM 452
▁RATHER 453
▁DEPEND 454
▁DOCUMENT 455
▁MODEL 456
▁ADVANCED 457
▁COMPANY 458
▁VIDEO 459
▁STANDARD 460
▁CERTAIN 461
▁DECISION 462
▁SPONGY 463
▁TITANIUM 464
▁TECHNICAL 465
▁AWAY 466
▁SEPARATE 467
▁BATTERIES 468
▁CHEAP 469
▁SWITCH 470
▁ROOM 471
▁SOURCE 472
▁EVALUATION 473
▁EMAIL 474
▁EITHER 475
▁BLACK 476
▁NORMAL 477
▁PROCESS 478
▁DIFFICULT 479
▁SELECT 480
▁EVERYBODY 481
▁UTTERANCE 482
▁PLACE 483
▁GENERAL 484
▁PROTOTYPE 485
▁REMEMBER 486
▁FOUND 487
▁MICROPHONE 488
▁UNDERSTAND 489
▁FINISH 490
▁WRITE 491
▁CURRENT 492
▁INSTEAD 493
▁CONSIDER 494
▁HAPPEN 495
▁RESEARCH 496
▁LEVEL 497
▁CHOOSE 498
Q 499
#0 500
#1 501