enigma-1.5b / tokenizer /__init__.py
shivendrra's picture
added tokenizer files
f5eb6b9 verified
raw
history blame
140 Bytes
from .main import DNAtokenizer
from .perChar import PerCharTokenizer
from .kmer_bpe import KmerPairTokenizer
from .kmer import KMerTokenizer