Spaces:
Sleeping
Sleeping
File size: 2,689 Bytes
05922fb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import argparse
import multiprocessing as mp
import os
import xml.etree.ElementTree as et
from io import StringIO
import nltk
buggy_annotations = [6538624, 6550700, 6547918, 6521702, 6541530, 6774318, 4531088, 4531238]
def iterate_docs(folder_path, tmp_path):
os.makedirs(tmp_path, exist_ok=True)
ft_path = os.path.join(folder_path, 'fulltext')
all_docs = list(filter(lambda x: x.endswith('.xml'), os.listdir(ft_path)))
for doc_name in all_docs:
it = et.iterparse(StringIO(open(os.path.join(ft_path, doc_name)).read()))
for _, el in it:
prefix, has_namespace, postfix = el.tag.partition('}')
if has_namespace:
el.tag = postfix
root = it.root
for sentence in root:
for annotation in sentence:
if "ID" in annotation.attrib and int(annotation.attrib['ID']) in buggy_annotations:
print('Delete one buggy annotation from', doc_name)
sentence.remove(annotation)
break
dump_path = os.path.join(tmp_path, doc_name)
et.ElementTree(root).write(dump_path)
# doc_xml.write(dump_path, default_namespace='')
yield dump_path
def process_doc(script_folder, doc_path):
os.chdir(script_folder)
print(f'processing {doc_path}...')
cmd = f'perl fttosem.pl {doc_path}'
print(cmd)
os.system(cmd)
print('Done')
return True
def main():
parser = argparse.ArgumentParser('Run fttosem perl examples.')
parser.add_argument('-s', help='script folder', type=str, required=True)
parser.add_argument('-p', help='path to corpora', type=str)
parser.add_argument('-o', help='output path', type=str, default='/tmp/framenet')
args = parser.parse_args()
script_folder = args.s
corpora_folder = args.p
if corpora_folder is None:
nltk.download('framenet')
corpora_folder = os.path.join(nltk.data.path[0], 'corpora', 'framenet_v17')
os.chdir(script_folder)
fns = list(iterate_docs(corpora_folder, args.o))
print(f'{len(fns)} documents detected.')
processes = list()
for fn in fns:
print('Processing', fn)
process = mp.Process(target=process_doc, args=(script_folder, fn))
process.start()
processes.append(process)
for process in processes:
process.join(timeout=480)
rst = os.listdir(args.o)
rst = list(filter(lambda x: x.endswith('.sem'), rst))
print(f'{len(rst)} Done.')
rst = [fn[:-4] for fn in rst]
fns = [fn[:-4] for fn in fns]
print('Unfinished docs:')
for fn in set(fns) - set(rst):
print(fn)
if __name__ == '__main__':
main()
|