commit files to HF hub

mecab_tokenizer.py

@@ -57,9 +57,12 @@ class MeCabTokenizer(PreTrainedTokenizer):
             # 最初と最後は空文字
             if result_word.hyosokei == "":
                 continue
-            if self.target_hinshi is not None and result_word.hinshi in self.target_hinshi:
-                # 特定の品詞のみ返します。
-                out.append(result_word.hyosokei)
+            if self.target_hinshi is not None:
+                if result_word.hinshi in self.target_hinshi:
+                    # 特定の品詞のみ返します。
+                    out.append(result_word.hyosokei)
+                else:
+                    continue
             else:
                 out.append(result_word.hyosokei)
         return out