File size: 636 Bytes
db6f2ff |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
# test tokenizer encode & decode consistency
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('/tokenizer_exp/other_tokenizer_vocab/hy', local_files_only=True, trust_remote_code=True)
test_data = [line.strip() for line in open('/tokenizer_exp/data/test.txt', 'r').readlines()]
num_origi_len = 0
num_token_len = 0
for d in test_data:
a = tokenizer.encode(d)
num_origi_len += len(d)
num_token_len += len(a)
b = tokenizer.decode(a)
assert b == d, f"encode & decode not consistent: {d} vs {b}"
print(f" original length: {num_origi_len}")
print(f" token length: {num_token_len}")
|