JessyTsu1 commited on
Commit
7ac2044
1 Parent(s): bf1e265

Upload tokenization_baichuan.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenization_baichuan.py +251 -0
tokenization_baichuan.py ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 Baichuan Inc. All Rights Reserved.
2
+
3
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
6
+ # and OPT implementations in this library. It has been modified from its
7
+ # original forms to accommodate minor architectural differences compared
8
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the "License");
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at
13
+ #
14
+ # http://www.apache.org/licenses/LICENSE-2.0
15
+ #
16
+ # Unless required by applicable law or agreed to in writing, software
17
+ # distributed under the License is distributed on an "AS IS" BASIS,
18
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
+ # See the License for the specific language governing permissions and
20
+ # limitations under the License.
21
+
22
+ import os
23
+ from shutil import copyfile
24
+ from typing import Any, Dict, List, Optional, Tuple
25
+
26
+ import sentencepiece as spm
27
+
28
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
29
+ from transformers.utils import logging
30
+
31
+
32
+ logger = logging.get_logger(__name__)
33
+
34
+ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
35
+
36
+ PRETRAINED_VOCAB_FILES_MAP = {
37
+ "vocab_file": {},
38
+ "tokenizer_file": {},
39
+ }
40
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
41
+
42
+
43
+ class BaichuanTokenizer(PreTrainedTokenizer):
44
+ """
45
+ Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
46
+
47
+ Args:
48
+ vocab_file (`str`):
49
+ Path to the vocabulary file.
50
+ """
51
+
52
+ vocab_files_names = VOCAB_FILES_NAMES
53
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
54
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
55
+ model_input_names = ["input_ids", "attention_mask"]
56
+
57
+ def __init__(
58
+ self,
59
+ vocab_file,
60
+ unk_token="<unk>",
61
+ bos_token="<s>",
62
+ eos_token="</s>",
63
+ pad_token=None,
64
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
65
+ add_bos_token=True,
66
+ add_eos_token=False,
67
+ clean_up_tokenization_spaces=False,
68
+ **kwargs,
69
+ ):
70
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
71
+ bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
72
+ eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
73
+ unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
74
+ pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
75
+ self.vocab_file = vocab_file
76
+ self.add_bos_token = add_bos_token
77
+ self.add_eos_token = add_eos_token
78
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
79
+ self.sp_model.Load(vocab_file)
80
+ super().__init__(
81
+ bos_token=bos_token,
82
+ eos_token=eos_token,
83
+ unk_token=unk_token,
84
+ pad_token=pad_token,
85
+ add_bos_token=add_bos_token,
86
+ add_eos_token=add_eos_token,
87
+ sp_model_kwargs=self.sp_model_kwargs,
88
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
89
+ **kwargs,
90
+ )
91
+
92
+ def __getstate__(self):
93
+ state = self.__dict__.copy()
94
+ state["sp_model"] = None
95
+ return state
96
+
97
+ def __setstate__(self, d):
98
+ self.__dict__ = d
99
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
100
+ self.sp_model.Load(self.vocab_file)
101
+
102
+ @property
103
+ def vocab_size(self):
104
+ """Returns vocab size"""
105
+ return self.sp_model.get_piece_size()
106
+
107
+ def get_vocab(self):
108
+ """Returns vocab as a dict"""
109
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
110
+ vocab.update(self.added_tokens_encoder)
111
+ return vocab
112
+
113
+ def _tokenize(self, text):
114
+ """Returns a tokenized string."""
115
+ return self.sp_model.encode(text, out_type=str)
116
+
117
+ def _convert_token_to_id(self, token):
118
+ """Converts a token (str) in an id using the vocab."""
119
+ return self.sp_model.piece_to_id(token)
120
+
121
+ def _convert_id_to_token(self, index):
122
+ """Converts an index (integer) in a token (str) using the vocab."""
123
+ token = self.sp_model.IdToPiece(index)
124
+ return token
125
+
126
+ def convert_tokens_to_string(self, tokens):
127
+ """Converts a sequence of tokens (string) in a single string."""
128
+ current_sub_tokens = []
129
+ out_string = ""
130
+ prev_is_special = False
131
+ for i, token in enumerate(tokens):
132
+ # make sure that special tokens are not decoded using sentencepiece model
133
+ if token in self.all_special_tokens:
134
+ if not prev_is_special and i != 0:
135
+ out_string += " "
136
+ out_string += self.sp_model.decode(current_sub_tokens) + token
137
+ prev_is_special = True
138
+ current_sub_tokens = []
139
+ else:
140
+ current_sub_tokens.append(token)
141
+ prev_is_special = False
142
+ out_string += self.sp_model.decode(current_sub_tokens)
143
+ return out_string
144
+
145
+ def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
146
+ """
147
+ Save the vocabulary and special tokens file to a directory.
148
+
149
+ Args:
150
+ save_directory (`str`):
151
+ The directory in which to save the vocabulary.
152
+
153
+ Returns:
154
+ `Tuple(str)`: Paths to the files saved.
155
+ """
156
+ if not os.path.isdir(save_directory):
157
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
158
+ return
159
+ out_vocab_file = os.path.join(
160
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
161
+ )
162
+
163
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
164
+ copyfile(self.vocab_file, out_vocab_file)
165
+ elif not os.path.isfile(self.vocab_file):
166
+ with open(out_vocab_file, "wb") as fi:
167
+ content_spiece_model = self.sp_model.serialized_model_proto()
168
+ fi.write(content_spiece_model)
169
+
170
+ return (out_vocab_file,)
171
+
172
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
173
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
174
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
175
+
176
+ output = bos_token_id + token_ids_0 + eos_token_id
177
+
178
+ if token_ids_1 is not None:
179
+ output = output + bos_token_id + token_ids_1 + eos_token_id
180
+
181
+ return output
182
+
183
+ def get_special_tokens_mask(
184
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
185
+ ) -> List[int]:
186
+ """
187
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
188
+ special tokens using the tokenizer `prepare_for_model` method.
189
+
190
+ Args:
191
+ token_ids_0 (`List[int]`):
192
+ List of IDs.
193
+ token_ids_1 (`List[int]`, *optional*):
194
+ Optional second list of IDs for sequence pairs.
195
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
196
+ Whether or not the token list is already formatted with special tokens for the model.
197
+
198
+ Returns:
199
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
200
+ """
201
+ if already_has_special_tokens:
202
+ return super().get_special_tokens_mask(
203
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
204
+ )
205
+
206
+ bos_token_id = [1] if self.add_bos_token else []
207
+ eos_token_id = [1] if self.add_eos_token else []
208
+
209
+ if token_ids_1 is None:
210
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
211
+ return (
212
+ bos_token_id
213
+ + ([0] * len(token_ids_0))
214
+ + eos_token_id
215
+ + bos_token_id
216
+ + ([0] * len(token_ids_1))
217
+ + eos_token_id
218
+ )
219
+
220
+ def create_token_type_ids_from_sequences(
221
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
222
+ ) -> List[int]:
223
+ """
224
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
225
+ sequence pair mask has the following format:
226
+
227
+ ```
228
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
229
+ | first sequence | second sequence |
230
+ ```
231
+
232
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
233
+
234
+ Args:
235
+ token_ids_0 (`List[int]`):
236
+ List of ids.
237
+ token_ids_1 (`List[int]`, *optional*):
238
+ Optional second list of IDs for sequence pairs.
239
+
240
+ Returns:
241
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
242
+ """
243
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
244
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
245
+
246
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
247
+
248
+ if token_ids_1 is not None:
249
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
250
+
251
+ return output