Upload test.py with huggingface_hub
Browse files
test.py
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from tokenizers import ByteLevelBPETokenizer
|
2 |
+
from transformers import AutoTokenizer
|
3 |
+
|
4 |
+
# Step 1: Initialize ByteLevelBPETokenizer
|
5 |
+
#tokenizer = ByteLevelBPETokenizer(
|
6 |
+
# "vocab.json",
|
7 |
+
# "merges.txt"
|
8 |
+
#)
|
9 |
+
|
10 |
+
# Step 2: Save the tokenizer configuration
|
11 |
+
#tokenizer.save_model("auto_model")
|
12 |
+
|
13 |
+
# Step 3: Load the tokenizer using AutoTokenizer
|
14 |
+
auto_tokenizer = AutoTokenizer.from_pretrained("./", use_fast=False, trust_remote_code=True)
|
15 |
+
|
16 |
+
# Test the tokenizer
|
17 |
+
text = "Hello, world!"
|
18 |
+
encoded = auto_tokenizer.encode(text)
|
19 |
+
decoded = auto_tokenizer.decode(encoded)
|
20 |
+
|
21 |
+
print("Encoded:", encoded)
|
22 |
+
print("Decoded:", decoded)
|
23 |
+
|
24 |
+
messages = [
|
25 |
+
{"role": "system", "content": "You are a helpful assistant."},
|
26 |
+
{"role": "user", "content": "Hello, how are you?"},
|
27 |
+
{"role": "assistant", "content": "I'm good, thank you! How can I help you today?"},
|
28 |
+
{"role": "user", "content": "Nothing"},
|
29 |
+
]
|
30 |
+
|
31 |
+
print('messages:', messages)
|
32 |
+
ids = auto_tokenizer.apply_chat_template(messages)
|
33 |
+
print(f"input_ids:\t{ids}")
|
34 |
+
text = auto_tokenizer.decode(ids)
|
35 |
+
print(f"input_text:\t[{text}]")
|