aarontseng/ybs-mt-en-zh

How to use

git lfs install
git clone https://huggingface.co./aarontseng/ybs-mt-en-zh

pip install ctranslate2
pip install sentencepiece

Basic Usage

import ctranslate2
import sentencepiece

src_model = sentencepiece.SentencePieceProcessor()
src_model.load("ybs-mt-en-zh/source.model")
tgt_model = sentencepiece.SentencePieceProcessor()
tgt_model.load("ybs-mt-en-zh/target.model")

translator = ctranslate2.Translator("ybs-mt-en-zh", device="cuda")  # "cpu" or "cuda"

encoded_line = src_model.encode_as_pieces("input text")

results = translator.translate_batch([encoded_line], batch_type="tokens", max_batch_size=64)

decoded_line = tgt_model.decode(results[0].hypotheses[0])

print(decoded_line)

Batch translation

import ctranslate2
import sentencepiece

src_path = "input.txt"
tgt_path = "output.txt"

src_model = sentencepiece.SentencePieceProcessor()
src_model.load("ybs-mt-en-zh/source.model")
tgt_model = sentencepiece.SentencePieceProcessor()
tgt_model.load("ybs-mt-en-zh/target.model")

translator = ctranslate2.Translator("ybs-mt-en-zh", device="cuda")  # "cpu" or "cuda"

src_file = open(src_path, 'r', encoding="utf-8")
src_lines = src_file.readlines()

encoded_lines = src_model.encode_as_pieces(src_lines)

results = translator.translate_batch(encoded_lines, batch_type="tokens", max_batch_size=1024)
translations = [translation.hypotheses[0] for translation in results]

decoded_lines = tgt_model.decode(translations)

tgt_file = open(tgt_path, "w", encoding="utf-8", newline='')

for line in decoded_lines:
    tgt_file.write(line)
    tgt_file.write('\n')