Ragdoll / tokenizer_.vocab
abhaskumarsinha's picture
added tokenizer
4e92eb8
<unk> 0
<s> 0
</s> 0
-2.98281
s -3.25482
, -3.64163
▁the -3.83505
e -3.92426
. -3.98345
-4.01794
t -4.03666
n -4.12316
▁a -4.17113
▁to -4.17459
d -4.36959
▁in -4.41674
y -4.56904
ing -4.59878
and -4.61393
▁she -4.77197
r -4.83711
ed -4.84531
a -4.87495
re -4.92487
▁of -4.93585
u -4.94813
ll -5.09773
- -5.11614
l -5.17304
▁it -5.24811
er -5.25543
▁is -5.26249
es -5.27771
om -5.28957
-5.35225
▁you -5.35663
▁b -5.39216
o -5.45565
▁on -5.46072
▁T -5.51677
▁be -5.52237
▁I -5.56146
f -5.56416
▁s -5.57973
▁m -5.58765
▁S -5.60246
▁an -5.65461
▁do -5.65883
▁f -5.66498
▁her -5.67479
▁w -5.69817
an -5.74529
▁t -5.78632
ar -5.79524
O -5.79603
m -5.80705
▁bu -5.81819
▁fr -5.84569
E -5.86177
▁who -5.866
▁that -5.86882
▁wi -5.87169
ade -5.89909
en -5.94578
▁c -5.96195
▁A -5.97429
al -6.00744
ver -6.01365
▁whe -6.0397
▁da -6.04174
tion -6.04637
un -6.04731
as -6.04809
st -6.05162
in -6.05191
nd -6.05331
w -6.0684
ill -6.08554
mos -6.08823
▁th -6.09591
▁for -6.10153
R -6.11177
tori -6.11304
▁home -6.1136
▁over -6.11539
▁She -6.11858
or -6.12376
ity -6.12455
ke -6.13723
▁fac -6.14681
ld -6.15102
ou -6.15562
th -6.16189
▁cl -6.16717
▁g -6.18264
▁li -6.18711
▁O -6.20291
▁he -6.22143
▁p -6.22893
▁no -6.24815
g -6.28119
ster -6.28227
▁n -6.28711
thing -6.28912
ever -6.2906
▁ye -6.30007
▁ga -6.30476
ch -6.31662
i -6.32376
▁con -6.34465
A -6.35218
▁l -6.36969
▁h -6.37459
ex -6.37778
▁co -6.38985
on -6.39749
▁C -6.4007
he -6.40221
▁o -6.42004
▁i -6.4279
le -6.44386
▁V -6.44511
P -6.44511
▁W -6.44511
B -6.44511
: -6.44511
? -6.44511
N -6.44511
z -6.44511
▁Znidarcic -6.44511
▁conflict -6.44511
ntain -6.44533
▁fir -6.44924
▁wor -6.45161
▁sell -6.45165
▁car -6.46197
ow -6.4632
rough -6.4642
▁up -6.47421
▁my -6.47499
bs -6.4795
ike -6.48774
," -6.48967
ce -6.49495
tle -6.49884
▁mo -6.50425
▁Car -6.51001
▁F -6.5113
▁bl -6.51221
▁dis -6.5152
av -6.52911
▁la -6.52958
king -6.53387
ter -6.53917
ty -6.54434
uth -6.55038
dow -6.55423
ile -6.5561
▁us -6.5635
ther -6.58487
ack -6.59476
▁sh -6.64105
ime -6.64708
▁ha -6.65268
day -6.66877
▁we -6.67527
▁M -6.68209
der -6.68547
▁com -6.69811
ra -6.70012
ri -6.70384
▁fa -6.71119
ant -6.7162
one -6.72907
mp -6.7366
k -6.74416
▁un -6.75502
ic -6.75614
h -6.7633
▁ne -6.77204
▁si -6.77776
ay -6.78566
ru -6.80931
▁ca -6.82626
ig -6.82691
ment -6.85812
ho -6.85835
ca -6.86862
H -6.86869
il -6.86978
at -6.87155
I -6.87438
to -6.87499
T -6.87566
ive -6.87875
li -6.88518
tche -6.88538
ine -6.89179
um -6.90013
use -6.92237
me -6.9249
pe -6.92835
An -6.93263
ut -6.93648
arme -6.93797
j -6.94511
-6.94511
▁Uganda -6.94511
▁photograph -6.94511
▁managed -6.94511
▁piano -6.94512
▁press -6.94519
▁Father -6.94529
▁val -6.94561
▁spa -6.94591
▁dark -6.94593
▁winter -6.94611
appe -6.94627
change -6.94631
ious -6.94748
▁camp -6.94855
ng -6.94956
rden -6.94978
▁di -6.95159
cu -6.95239
▁let -6.95528
term -6.95542
▁sof -6.95617
▁", -6.95629
ights -6.9572
▁some -6.95731
place -6.958
▁wind -6.96054
▁thi -6.96182
▁pu -6.96681
▁comm -6.96754
desc -6.9678
▁po -6.96809
fe -6.97021
ome -6.97134
dent -6.97519
▁G -6.97692
▁L -6.97693
▁forg -6.97694
▁essay -6.97879
▁out -6.97915
▁child -6.97955
▁per -6.97978
▁room -6.98075
▁his -6.98661
ol -6.98952
▁pla -6.99025
ances -6.99299
ard -6.99428
▁dr -7.00139
all -7.00655
▁H -7.02766
▁ar -7.03612
▁free -7.04492
▁hi -7.04835
ts -7.05165
ion -7.05786
ge -7.06057
▁ex -7.06557
ve -7.06592
ture -7.08079
▁gr -7.08256
nk -7.12309
b -7.12789
▁par -7.15886
ong -7.17102
arm -7.17354
▁wit -7.18408
▁ge -7.1933
▁The -7.19351
▁e -7.21379
ad -7.21474
amil -7.23763
si -7.24104
ess -7.24262
eb -7.25031
erve -7.26669
▁wh -7.27789
▁live -7.28473
cho -7.28653
▁d -7.29344
▁Mo -7.29911
te -7.3141
tra -7.31663
itch -7.31822
▁lit -7.3428
till -7.35067
ind -7.36005
men -7.37029
p -7.38937
c -7.40623
▁se -7.42879
▁tr -7.43108
ation -7.4369
mil -7.45181
ki -7.45207
▁sho -7.47937
▁acc -7.48153
▁wa -7.48241
ti -7.4852
ass -7.49028
▁st -7.49096
se -7.50074
▁In -7.53188
▁ho -7.54006
old -7.54322
other -7.56929
ors -7.58642
are -7.59943
mm -7.61377
ves -7.63739
▁Al -7.63808
tin -7.65018
eme -7.661
ro -7.67242
it -7.6764
ook -7.68149
▁mi -7.70738
ct -7.72334
ast -7.73492
ck -7.73984
ure -7.77089
us -7.77799
▁y -7.78405
trac -7.7845
id -7.78587
ul -7.80499
lo -7.8428
end -7.8498
F -7.85778
child -7.86205
G -7.86792
L -7.86792
essay -7.87043
room -7.874
C -7.87462
of -7.87688
whi -7.89978
iv -7.90342
cco -7.90462
" -7.90846
ent -7.90927
up -7.91017
lit -7.91248
ff -7.91432
ma -7.91669
ell -7.91905
ati -7.92941
bo -7.93198
bl -7.93333
mas -7.93684
▁ti -7.9427
D -7.94511
) -7.94511
; -7.94511
U -7.94511
( -7.94511
ugh -7.94528
pp -7.95039
hoto -7.95116
mb -7.95275
ivi -7.9545
v -7.96254
et -7.96412
pre -7.96724
alm -7.96927
amp -7.97751
rap -7.98387
ark -7.98456
ough -7.98494
bi -7.98855
ug -8.00002
lar -8.00274
ear -8.02483
▁re -8.02483
ight -8.02664
side -8.04561
sel -8.05147
▁pi -8.05401
rand -8.05666
our -8.06317
mer -8.07982
iss -8.08135
oth -8.08572
ot -8.08918
ga -8.09034
ree -8.09464
is -8.10217
cen -8.10548
▁so -8.10812
ian -8.11603
nter -8.12159
lve -8.12475
no -8.13398
▁ro -8.13407
ite -8.13992
ur -8.15648
eed -8.15933
ous -8.16485
ath -8.17035
int -8.17256
ana -8.2029
sce -8.22229
ir -8.22503
▁sa -8.26216
nti -8.28612
tan -8.28859
▁ac -8.30665
oo -8.36908
ides -8.37012
ove -8.41417
▁sta -8.43306
stra -8.48403
ons -8.4944
▁wil -8.56631
ni -8.60037
lle -8.63056
und -8.64127
lay -8.67965
▁bro -8.70233
hat -8.72064
ame -8.72281
comm -8.75146
▁cam -8.75814
▁cont -8.77115
str -8.87536
unt -8.89135
▁r -8.94977
am -8.9529
oi -8.95627
▁le -8.96696
su -8.96796
▁pa -9.01054
hou -9.06986
oda -9.07227
mo -9.08273
act -9.08564
▁Sh -9.09262
mili -9.1038
▁br -9.14123
▁su -9.22683
rs -9.27548
Z -9.28681
W -9.28691
V -9.28701
M -9.28711
x -9.28721
S -9.28731
ange -9.28731