paulhindemith commited on
Commit
9841823
1 Parent(s): 84d2513

commit files to HF hub

Browse files
Files changed (1) hide show
  1. mecab_tokenizer.py +7 -29
mecab_tokenizer.py CHANGED
@@ -9,14 +9,6 @@ class MeCabResult(NamedTuple):
9
  """
10
  hyosokei: str
11
  hinshi: str
12
- hinshi_saibunrui_1: str
13
- hinshi_saibunrui_2: str
14
- hinshi_saibunrui_3: str
15
- katsuyokei_1: str
16
- katsuyokei_2: str
17
- genkei: str = ""
18
- yomi: str = ""
19
- hatsuon: str = ""
20
 
21
 
22
  class MeCabTokenizer(PreTrainedTokenizer):
@@ -34,12 +26,16 @@ class MeCabTokenizer(PreTrainedTokenizer):
34
 
35
  self.target_hinshi = hinshi
36
  if mecab_dicdir is not None:
37
- self.mecab = MeCab.Tagger(f"-d {mecab_dicdir} -O '' -F '%m,%H\n'")
 
38
  else:
39
- self.mecab = MeCab.Tagger("-O '' -F '%m,%H\n'")
40
 
41
  super().__init__(**kwargs)
42
 
 
 
 
43
  def _tokenize(self, text: str) -> list[str]:
44
  """文章から特定の品詞の単語を返します。
45
 
@@ -77,29 +73,11 @@ class MeCabTokenizer(PreTrainedTokenizer):
77
  list[MeCabResult]: MeCabの解析結果
78
  """
79
  nodes = self.mecab.parse(text).split("\n")
80
- #形態素1つ1つを処理
81
  out = []
82
  for node in nodes:
83
  args = node.split(",")
84
  if args[0] in ["EOS", ""]:
85
  continue
86
- # 辞書によって異なる
87
- if len(args) == 10:
88
- mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
89
- args[4], args[5], args[6], args[7],
90
- args[8], args[9])
91
- elif len(args) == 7:
92
- # 英語
93
- mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
94
- args[4], args[5], args[6], "*", "*",
95
- "*")
96
- elif len(args) == 27:
97
- # 補助記号
98
- mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
99
- args[4], args[5], args[6], "*", "*",
100
- "*")
101
- else:
102
- raise NotImplementedError(
103
- f"unsupposed parse args_length: {len(args)}, args: {args}")
104
  out.append(mecab_result)
105
  return out
 
9
  """
10
  hyosokei: str
11
  hinshi: str
 
 
 
 
 
 
 
 
12
 
13
 
14
  class MeCabTokenizer(PreTrainedTokenizer):
 
26
 
27
  self.target_hinshi = hinshi
28
  if mecab_dicdir is not None:
29
+ self.mecab = MeCab.Tagger(
30
+ f"-d {mecab_dicdir} -O '' -F '%m,%f[0]\n'")
31
  else:
32
+ self.mecab = MeCab.Tagger("-O '' -F '%m,%f[0]\n'")
33
 
34
  super().__init__(**kwargs)
35
 
36
+ def set_dicdir(self, mecab_dicdir: str):
37
+ self.mecab = MeCab.Tagger(f"-d {mecab_dicdir} -O '' -F '%m,%f[0]\n'")
38
+
39
  def _tokenize(self, text: str) -> list[str]:
40
  """文章から特定の品詞の単語を返します。
41
 
 
73
  list[MeCabResult]: MeCabの解析結果
74
  """
75
  nodes = self.mecab.parse(text).split("\n")
 
76
  out = []
77
  for node in nodes:
78
  args = node.split(",")
79
  if args[0] in ["EOS", ""]:
80
  continue
81
+ mecab_result = MeCabResult(args[0], args[1])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  out.append(mecab_result)
83
  return out