speecht5_tts_jvs_ver1_e20_openjtalk_longer_20230809-031157_tokenizer

speecht5_openjtalk_tokenizer.py

@@ -9,6 +9,9 @@ from itertools import chain
 from typing import List, Optional
 
 
+NP_CHARCTERS = " !\"#$%&'()=~|`{+*}<>?_-^\\@[;:],./　！”＃＄％＆’（）＝～｜｀｛＋＊｝＜＞？＿ー＾￥＠「；：」、。・`"
+
+
 def _g2p_with_np(text: str, np_lsit: str) -> List[str]:
     from pyopenjtalk import g2p
 
@@ -25,15 +28,11 @@ def _g2p_with_np(text: str, np_lsit: str) -> List[str]:
     )
 
 
-NP_CHARCTERS = " !\"#$%&'()=~|`{+*}<>?_-^\\@[;:],./　！”＃＄％＆’（）＝～｜｀｛＋＊｝＜＞？＿ー＾￥＠「；：」、。・`"
-
-
 class SpeechT5OpenjtalkTokenizer(SpeechT5Tokenizer):
-    vocab_files_names = {"vocab_file": "spm_char.model"}
+    vocab_files_names = {"vocab_file": "vocab.json"}
     pretrained_vocab_files_map = {}
     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["input_ids", "attention_mask"]
-    label2id = {}
 
     def __init__(
         self,
@@ -58,6 +57,7 @@ class SpeechT5OpenjtalkTokenizer(SpeechT5Tokenizer):
             pass
 
         self.non_phenome_characters = non_phenome_characters
+        self.vocab_file = vocab_file
 
         if isinstance(vocab_file, str) and vocab_file.endswith(".json"):
             with open(vocab_file, encoding="utf-8") as f:
@@ -75,6 +75,11 @@ class SpeechT5OpenjtalkTokenizer(SpeechT5Tokenizer):
     def get_vocab(self):
         return self.label2id
 
+    def __getstate__(self):
+        state = super().__getstate__()
+        del state["sp_model"]
+        return state
+
     def save_vocabulary(
         self, save_directory: str, filename_prefix: Optional[str] = None
     ):