extract_sents.py

from typing import TextIO
import re

import click
import conllu
import jsonlines


@click.command(help="Extract a parallel corpus from a CoNLL-U file with translations")
@click.argument("conllu_path", type=click.File("r"))
@click.argument("output_path", type=click.File("w"), default="-")
@click.option("--main-langcode", default="br", show_default=True)
@click.option("--require-langcode", multiple=True, show_default=True)
def main(
    conllu_path: TextIO,
    main_langcode: str,
    output_path: TextIO,
    require_langcode: list[str],
):
    with jsonlines.Writer(output_path) as out_stream:
        for tokenlist in conllu.parse_incr(conllu_path):
            if m := re.match(r"'?(?P<content>[^/]+?)'?$", tokenlist.metadata["text"]):
                main_text = m.group("content")
            else:
                continue
            translations = {
                km.group("langcode"): kv.group("content")
                for k, v in tokenlist.metadata.items()
                if (km := re.match(r"text_(?P<langcode>.*)", k))
                and (kv := re.match(r"'?(?P<content>[^/]+?)'?$", v))
            }
            if not all(l in translations for l in require_langcode):
                continue
            out_stream.write(
                {
                    "translation": {
                        main_langcode: main_text,
                        **translations,
                    }
                }
            )


if __name__ == "__main__":
    main()