woodchen7 commited on
Commit
07eef72
1 Parent(s): 113da9b

Upload test.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. test.py +35 -0
test.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tokenizers import ByteLevelBPETokenizer
2
+ from transformers import AutoTokenizer
3
+
4
+ # Step 1: Initialize ByteLevelBPETokenizer
5
+ #tokenizer = ByteLevelBPETokenizer(
6
+ # "vocab.json",
7
+ # "merges.txt"
8
+ #)
9
+
10
+ # Step 2: Save the tokenizer configuration
11
+ #tokenizer.save_model("auto_model")
12
+
13
+ # Step 3: Load the tokenizer using AutoTokenizer
14
+ auto_tokenizer = AutoTokenizer.from_pretrained("./", use_fast=False, trust_remote_code=True)
15
+
16
+ # Test the tokenizer
17
+ text = "Hello, world!"
18
+ encoded = auto_tokenizer.encode(text)
19
+ decoded = auto_tokenizer.decode(encoded)
20
+
21
+ print("Encoded:", encoded)
22
+ print("Decoded:", decoded)
23
+
24
+ messages = [
25
+ {"role": "system", "content": "You are a helpful assistant."},
26
+ {"role": "user", "content": "Hello, how are you?"},
27
+ {"role": "assistant", "content": "I'm good, thank you! How can I help you today?"},
28
+ {"role": "user", "content": "Nothing"},
29
+ ]
30
+
31
+ print('messages:', messages)
32
+ ids = auto_tokenizer.apply_chat_template(messages)
33
+ print(f"input_ids:\t{ids}")
34
+ text = auto_tokenizer.decode(ids)
35
+ print(f"input_text:\t[{text}]")