Spaces:

seyoungsong
/

flores101_mm100_175M

Running

App Files Files Community

seyoungsong commited on Nov 26, 2023

Commit

68fdd8d

•

1 Parent(s): 5b36141

r

Browse files

Files changed (11) hide show

model_files/.gitattributes +35 -0
model_files/.gitignore +203 -0
model_files/README.md +295 -0
model_files/added_tokens.json +126 -0
model_files/config.json +32 -0
model_files/generation_config.json +8 -0
model_files/model.safetensors +3 -0
model_files/sentencepiece.bpe.model +3 -0
model_files/special_tokens_map.json +133 -0
model_files/tokenizer_config.json +1167 -0
model_files/vocab.json +0 -0

model_files/.gitattributes ADDED Viewed

	@@ -0,0 +1,35 @@

+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

model_files/.gitignore ADDED Viewed

	@@ -0,0 +1,203 @@

+temp.*
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+# C extensions
+*.so
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+# Translations
+*.mo
+*.pot
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+# Flask stuff:
+instance/
+.webassets-cache
+# Scrapy stuff:
+.scrapy
+# Sphinx documentation
+docs/_build/
+# PyBuilder
+.pybuilder/
+target/
+# Jupyter Notebook
+.ipynb_checkpoints
+# IPython
+profile_default/
+ipython_config.py
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+# SageMath parsed files
+*.sage.py
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+# Spyder project settings
+.spyderproject
+.spyproject
+# Rope project settings
+.ropeproject
+# mkdocs documentation
+/site
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+# Pyre type checker
+.pyre/
+# pytype static type analyzer
+.pytype/
+# Cython debug symbols
+cython_debug/
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+# Icon must end with two \r
+Icon
+# Thumbnails
+._*
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+*~
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+# KDE directory preferences
+.directory
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*

model_files/README.md ADDED Viewed

	@@ -0,0 +1,295 @@

+---
+pipeline_tag: translation
+license: mit
+language:
+  - multilingual
+  - af
+  - am
+  - ar
+  - as
+  - ast
+  - ay
+  - az
+  - ba
+  - be
+  - bg
+  - bn
+  - br
+  - bs
+  - ca
+  - ceb
+  - cjk
+  - cs
+  - cy
+  - da
+  - de
+  - dyu
+  - el
+  - en
+  - es
+  - et
+  - fa
+  - ff
+  - fi
+  - fr
+  - fy
+  - ga
+  - gd
+  - gl
+  - gu
+  - ha
+  - he
+  - hi
+  - hr
+  - ht
+  - hu
+  - hy
+  - id
+  - ig
+  - ilo
+  - is
+  - it
+  - ja
+  - jv
+  - ka
+  - kac
+  - kam
+  - kea
+  - kg
+  - kk
+  - km
+  - kmb
+  - kmr
+  - kn
+  - ko
+  - ku
+  - ky
+  - lb
+  - lg
+  - ln
+  - lo
+  - lt
+  - luo
+  - lv
+  - mg
+  - mi
+  - mk
+  - ml
+  - mn
+  - mr
+  - ms
+  - mt
+  - my
+  - ne
+  - nl
+  - no
+  - ns
+  - ny
+  - oc
+  - om
+  - or
+  - pa
+  - pl
+  - ps
+  - pt
+  - qu
+  - ro
+  - ru
+  - sd
+  - shn
+  - si
+  - sk
+  - sl
+  - sn
+  - so
+  - sq
+  - sr
+  - ss
+  - su
+  - sv
+  - sw
+  - ta
+  - te
+  - tg
+  - th
+  - ti
+  - tl
+  - tn
+  - tr
+  - uk
+  - umb
+  - ur
+  - uz
+  - vi
+  - wo
+  - xh
+  - yi
+  - yo
+  - zh
+  - zu
+---
+# Flores101: Large-Scale Multilingual Machine Translation
+`flores101_mm100_175M` is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was introduced in this [paper](https://aclanthology.org/2022.tacl-1.30) and released in [this](https://github.com/facebookresearch/fairseq/tree/main/examples/flores101) repository.
+The model architecture and config are the same as [M2M100](https://huggingface.co/facebook/m2m100_418M) implementation, but the **tokenizer should be modified** to adjust language codes.
+```python
+from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
+hi_text = "जीवन एक चॉकलेट बॉक्स की तरह है।"
+chinese_text = "生活就像一盒巧克力。"
+model = M2M100ForConditionalGeneration.from_pretrained("seyoungsong/flores101_mm100_175M")
+tokenizer: M2M100Tokenizer = M2M100Tokenizer.from_pretrained("seyoungsong/flores101_mm100_175M")
+# FIX TOKENIZER!
+tokenizer.lang_token_to_id = {t: i for t, i in zip(tokenizer.all_special_tokens, tokenizer.all_special_ids) if i > 5}
+tokenizer.lang_code_to_token = {s.strip("_"): s for s in tokenizer.lang_token_to_id}
+tokenizer.lang_code_to_id = {s.strip("_"): i for s, i in tokenizer.lang_token_to_id.items()}
+tokenizer.id_to_lang_token = {i: s for s, i in tokenizer.lang_token_to_id.items()}
+# translate Hindi to French
+tokenizer.src_lang = "hi"
+encoded_hi = tokenizer(hi_text, return_tensors="pt")
+generated_tokens = model.generate(**encoded_hi, forced_bos_token_id=tokenizer.get_lang_id("fr"))
+tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+# => "La vie est comme une boîte de chocolat."
+# translate Chinese to English
+tokenizer.src_lang = "zh"
+encoded_zh = tokenizer(chinese_text, return_tensors="pt")
+generated_tokens = model.generate(**encoded_zh, forced_bos_token_id=tokenizer.get_lang_id("en"))
+tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+# => "Life is like a chocolate box."
+```
+## Languages covered
+| Language         | lang code |
+| ---------------- | --------- |
+| Akrikaans        | af        |
+| Amharic          | am        |
+| Arabic           | ar        |
+| Assamese         | as        |
+| Asturian         | ast       |
+| Aymara           | ay        |
+| Azerbaijani      | az        |
+| Bashkir          | ba        |
+| Belarusian       | be        |
+| Bulgarian        | bg        |
+| Bengali          | bn        |
+| Breton           | br        |
+| Bosnian          | bs        |
+| Catalan          | ca        |
+| Cebuano          | ceb       |
+| Chokwe           | cjk       |
+| Czech            | cs        |
+| Welsh            | cy        |
+| Danish           | da        |
+| German           | de        |
+| Dyula            | dyu       |
+| Greek            | el        |
+| English          | en        |
+| Spanish          | es        |
+| Estonian         | et        |
+| Persian          | fa        |
+| Fulah            | ff        |
+| Finnish          | fi        |
+| French           | fr        |
+| Western Frisian  | fy        |
+| Irish            | ga        |
+| Scottish Gaelic  | gd        |
+| Galician         | gl        |
+| Gujarati         | gu        |
+| Hausa            | ha        |
+| Hebrew           | he        |
+| Hindi            | hi        |
+| Croatian         | hr        |
+| Haitian Creole   | ht        |
+| Hungarian        | hu        |
+| Armenian         | hy        |
+| Indonesian       | id        |
+| Igbo             | ig        |
+| Iloko            | ilo       |
+| Icelandic        | is        |
+| Italian          | it        |
+| Japanese         | ja        |
+| Javanese         | jv        |
+| Georgian         | ka        |
+| Kachin           | kac       |
+| Kamba            | kam       |
+| Kabuverdianu     | kea       |
+| Kongo            | kg        |
+| Kazakh           | kk        |
+| Central Khmer    | km        |
+| Kimbundu         | kmb       |
+| Northern Kurdish | kmr       |
+| Kannada          | kn        |
+| Korean           | ko        |
+| Kurdish          | ku        |
+| Kyrgyz           | ky        |
+| Luxembourgish    | lb        |
+| Ganda            | lg        |
+| Lingala          | ln        |
+| Lao              | lo        |
+| Lithuanian       | lt        |
+| Luo              | luo       |
+| Latvian          | lv        |
+| Malagasy         | mg        |
+| Maori            | mi        |
+| Macedonian       | mk        |
+| Malayalam        | ml        |
+| Mongolian        | mn        |
+| Marathi          | mr        |
+| Malay            | ms        |
+| Maltese          | mt        |
+| Burmese          | my        |
+| Nepali           | ne        |
+| Dutch            | nl        |
+| Norwegian        | no        |
+| Northern Sotho   | ns        |
+| Nyanja           | ny        |
+| Occitan          | oc        |
+| Oromo            | om        |
+| Oriya            | or        |
+| Punjabi          | pa        |
+| Polish           | pl        |
+| Pashto           | ps        |
+| Portuguese       | pt        |
+| Quechua          | qu        |
+| Romanian         | ro        |
+| Russian          | ru        |
+| Sindhi           | sd        |
+| Shan             | shn       |
+| Sinhala          | si        |
+| Slovak           | sk        |
+| Slovenian        | sl        |
+| Shona            | sn        |
+| Somali           | so        |
+| Albanian         | sq        |
+| Serbian          | sr        |
+| Swati            | ss        |
+| Sundanese        | su        |
+| Swedish          | sv        |
+| Swahili          | sw        |
+| Tamil            | ta        |
+| Telugu           | te        |
+| Tajik            | tg        |
+| Thai             | th        |
+| Tigrinya         | ti        |
+| Tagalog          | tl        |
+| Tswana           | tn        |
+| Turkish          | tr        |
+| Ukrainian        | uk        |
+| Umbundu          | umb       |
+| Urdu             | ur        |
+| Uzbek            | uz        |
+| Vietnamese       | vi        |
+| Wolof            | wo        |
+| Xhosa            | xh        |
+| Yiddish          | yi        |
+| Yoruba           | yo        |
+| Chinese          | zh        |
+| Zulu             | zu        |

model_files/added_tokens.json ADDED Viewed

	@@ -0,0 +1,126 @@

+{
+  "__af__": 256001,
+  "__am__": 256002,
+  "__ar__": 256003,
+  "__as__": 256004,
+  "__ast__": 256005,
+  "__ay__": 256006,
+  "__az__": 256007,
+  "__ba__": 256008,
+  "__be__": 256009,
+  "__bg__": 256010,
+  "__bn__": 256011,
+  "__br__": 256012,
+  "__bs__": 256013,
+  "__ca__": 256014,
+  "__ceb__": 256015,
+  "__cjk__": 256016,
+  "__cs__": 256017,
+  "__cy__": 256018,
+  "__da__": 256019,
+  "__de__": 256020,
+  "__dyu__": 256021,
+  "__el__": 256022,
+  "__en__": 256023,
+  "__es__": 256024,
+  "__et__": 256025,
+  "__fa__": 256026,
+  "__ff__": 256027,
+  "__fi__": 256028,
+  "__fr__": 256029,
+  "__fy__": 256030,
+  "__ga__": 256031,
+  "__gd__": 256032,
+  "__gl__": 256033,
+  "__gu__": 256034,
+  "__ha__": 256035,
+  "__he__": 256036,
+  "__hi__": 256037,
+  "__hr__": 256038,
+  "__ht__": 256039,
+  "__hu__": 256040,
+  "__hy__": 256041,
+  "__id__": 256042,
+  "__ig__": 256043,
+  "__ilo__": 256044,
+  "__is__": 256045,
+  "__it__": 256046,
+  "__ja__": 256047,
+  "__jv__": 256048,
+  "__ka__": 256049,
+  "__kac__": 256050,
+  "__kam__": 256051,
+  "__kea__": 256052,
+  "__kg__": 256053,
+  "__kk__": 256054,
+  "__km__": 256055,
+  "__kmb__": 256056,
+  "__kmr__": 256057,
+  "__kn__": 256058,
+  "__ko__": 256059,
+  "__ku__": 256060,
+  "__ky__": 256061,
+  "__lb__": 256062,
+  "__lg__": 256063,
+  "__ln__": 256064,
+  "__lo__": 256065,
+  "__lt__": 256066,
+  "__luo__": 256067,
+  "__lv__": 256068,
+  "__mg__": 256069,
+  "__mi__": 256070,
+  "__mk__": 256071,
+  "__ml__": 256072,
+  "__mn__": 256073,
+  "__mr__": 256074,
+  "__ms__": 256075,
+  "__mt__": 256076,
+  "__my__": 256077,
+  "__ne__": 256078,
+  "__nl__": 256079,
+  "__no__": 256080,
+  "__ns__": 256081,
+  "__ny__": 256082,
+  "__oc__": 256083,
+  "__om__": 256084,
+  "__or__": 256085,
+  "__pa__": 256086,
+  "__pl__": 256087,
+  "__ps__": 256088,
+  "__pt__": 256089,
+  "__qu__": 256090,
+  "__ro__": 256091,
+  "__ru__": 256092,
+  "__sd__": 256093,
+  "__shn__": 256094,
+  "__si__": 256095,
+  "__sk__": 256096,
+  "__sl__": 256097,
+  "__sn__": 256098,
+  "__so__": 256099,
+  "__sq__": 256100,
+  "__sr__": 256101,
+  "__ss__": 256102,
+  "__su__": 256103,
+  "__sv__": 256104,
+  "__sw__": 256105,
+  "__ta__": 256106,
+  "__te__": 256107,
+  "__tg__": 256108,
+  "__th__": 256109,
+  "__ti__": 256110,
+  "__tl__": 256111,
+  "__tn__": 256112,
+  "__tr__": 256113,
+  "__uk__": 256114,
+  "__umb__": 256115,
+  "__ur__": 256116,
+  "__uz__": 256117,
+  "__vi__": 256118,
+  "__wo__": 256119,
+  "__xh__": 256120,
+  "__yi__": 256121,
+  "__yo__": 256122,
+  "__zh__": 256123,
+  "__zu__": 256124
+}

model_files/config.json ADDED Viewed

	@@ -0,0 +1,32 @@

+{
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "architectures": [
+    "M2M100ForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "d_model": 512,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 2,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0,
+  "encoder_layers": 6,
+  "eos_token_id": 2,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_position_embeddings": 1024,
+  "model_type": "m2m_100",
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "scale_embedding": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "use_cache": true,
+  "vocab_size": 256125
+}

model_files/generation_config.json ADDED Viewed

	@@ -0,0 +1,8 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.35.2"
+}

model_files/model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1561d6c592b727a7bb5d229e2d72730bf662d83f8f1915803ca6b35b3c8399d8
+size 701135520

model_files/sentencepiece.bpe.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6ee4dc054a17c18fe81f76c0b1cda00e9fc1cfd9e0f1a16cb6d77009e2076653
+size 4870365

model_files/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,133 @@

+{
+  "additional_special_tokens": [
+    "__af__",
+    "__am__",
+    "__ar__",
+    "__as__",
+    "__ast__",
+    "__ay__",
+    "__az__",
+    "__ba__",
+    "__be__",
+    "__bg__",
+    "__bn__",
+    "__br__",
+    "__bs__",
+    "__ca__",
+    "__ceb__",
+    "__cjk__",
+    "__cs__",
+    "__cy__",
+    "__da__",
+    "__de__",
+    "__dyu__",
+    "__el__",
+    "__en__",
+    "__es__",
+    "__et__",
+    "__fa__",
+    "__ff__",
+    "__fi__",
+    "__fr__",
+    "__fy__",
+    "__ga__",
+    "__gd__",
+    "__gl__",
+    "__gu__",
+    "__ha__",
+    "__he__",
+    "__hi__",
+    "__hr__",
+    "__ht__",
+    "__hu__",
+    "__hy__",
+    "__id__",
+    "__ig__",
+    "__ilo__",
+    "__is__",
+    "__it__",
+    "__ja__",
+    "__jv__",
+    "__ka__",
+    "__kac__",
+    "__kam__",
+    "__kea__",
+    "__kg__",
+    "__kk__",
+    "__km__",
+    "__kmb__",
+    "__kmr__",
+    "__kn__",
+    "__ko__",
+    "__ku__",
+    "__ky__",
+    "__lb__",
+    "__lg__",
+    "__ln__",
+    "__lo__",
+    "__lt__",
+    "__luo__",
+    "__lv__",
+    "__mg__",
+    "__mi__",
+    "__mk__",
+    "__ml__",
+    "__mn__",
+    "__mr__",
+    "__ms__",
+    "__mt__",
+    "__my__",
+    "__ne__",
+    "__nl__",
+    "__no__",
+    "__ns__",
+    "__ny__",
+    "__oc__",
+    "__om__",
+    "__or__",
+    "__pa__",
+    "__pl__",
+    "__ps__",
+    "__pt__",
+    "__qu__",
+    "__ro__",
+    "__ru__",
+    "__sd__",
+    "__shn__",
+    "__si__",
+    "__sk__",
+    "__sl__",
+    "__sn__",
+    "__so__",
+    "__sq__",
+    "__sr__",
+    "__ss__",
+    "__su__",
+    "__sv__",
+    "__sw__",
+    "__ta__",
+    "__te__",
+    "__tg__",
+    "__th__",
+    "__ti__",
+    "__tl__",
+    "__tn__",
+    "__tr__",
+    "__uk__",
+    "__umb__",
+    "__ur__",
+    "__uz__",
+    "__vi__",
+    "__wo__",
+    "__xh__",
+    "__yi__",
+    "__yo__",
+    "__zh__",
+    "__zu__"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

model_files/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,1167 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256001": {
+      "content": "__af__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256002": {
+      "content": "__am__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256003": {
+      "content": "__ar__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256004": {
+      "content": "__as__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256005": {
+      "content": "__ast__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256006": {
+      "content": "__ay__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256007": {
+      "content": "__az__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256008": {
+      "content": "__ba__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256009": {
+      "content": "__be__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256010": {
+      "content": "__bg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256011": {
+      "content": "__bn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256012": {
+      "content": "__br__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256013": {
+      "content": "__bs__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256014": {
+      "content": "__ca__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256015": {
+      "content": "__ceb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256016": {
+      "content": "__cjk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256017": {
+      "content": "__cs__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256018": {
+      "content": "__cy__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256019": {
+      "content": "__da__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256020": {
+      "content": "__de__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256021": {
+      "content": "__dyu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256022": {
+      "content": "__el__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256023": {
+      "content": "__en__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256024": {
+      "content": "__es__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256025": {
+      "content": "__et__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256026": {
+      "content": "__fa__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256027": {
+      "content": "__ff__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256028": {
+      "content": "__fi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256029": {
+      "content": "__fr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256030": {
+      "content": "__fy__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256031": {
+      "content": "__ga__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256032": {
+      "content": "__gd__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256033": {
+      "content": "__gl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256034": {
+      "content": "__gu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256035": {
+      "content": "__ha__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256036": {
+      "content": "__he__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256037": {
+      "content": "__hi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256038": {
+      "content": "__hr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256039": {
+      "content": "__ht__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256040": {
+      "content": "__hu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256041": {
+      "content": "__hy__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256042": {
+      "content": "__id__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256043": {
+      "content": "__ig__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256044": {
+      "content": "__ilo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256045": {
+      "content": "__is__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256046": {
+      "content": "__it__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256047": {
+      "content": "__ja__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256048": {
+      "content": "__jv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256049": {
+      "content": "__ka__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256050": {
+      "content": "__kac__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256051": {
+      "content": "__kam__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256052": {
+      "content": "__kea__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256053": {
+      "content": "__kg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256054": {
+      "content": "__kk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256055": {
+      "content": "__km__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256056": {
+      "content": "__kmb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256057": {
+      "content": "__kmr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256058": {
+      "content": "__kn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256059": {
+      "content": "__ko__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256060": {
+      "content": "__ku__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256061": {
+      "content": "__ky__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256062": {
+      "content": "__lb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256063": {
+      "content": "__lg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256064": {
+      "content": "__ln__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256065": {
+      "content": "__lo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256066": {
+      "content": "__lt__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256067": {
+      "content": "__luo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256068": {
+      "content": "__lv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256069": {
+      "content": "__mg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256070": {
+      "content": "__mi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256071": {
+      "content": "__mk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256072": {
+      "content": "__ml__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256073": {
+      "content": "__mn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256074": {
+      "content": "__mr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256075": {
+      "content": "__ms__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256076": {
+      "content": "__mt__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256077": {
+      "content": "__my__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256078": {
+      "content": "__ne__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256079": {
+      "content": "__nl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256080": {
+      "content": "__no__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256081": {
+      "content": "__ns__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256082": {
+      "content": "__ny__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256083": {
+      "content": "__oc__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256084": {
+      "content": "__om__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256085": {
+      "content": "__or__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256086": {
+      "content": "__pa__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256087": {
+      "content": "__pl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256088": {
+      "content": "__ps__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256089": {
+      "content": "__pt__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256090": {
+      "content": "__qu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256091": {
+      "content": "__ro__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256092": {
+      "content": "__ru__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256093": {
+      "content": "__sd__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256094": {
+      "content": "__shn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256095": {
+      "content": "__si__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256096": {
+      "content": "__sk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256097": {
+      "content": "__sl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256098": {
+      "content": "__sn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256099": {
+      "content": "__so__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256100": {
+      "content": "__sq__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256101": {
+      "content": "__sr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256102": {
+      "content": "__ss__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256103": {
+      "content": "__su__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256104": {
+      "content": "__sv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256105": {
+      "content": "__sw__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256106": {
+      "content": "__ta__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256107": {
+      "content": "__te__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256108": {
+      "content": "__tg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256109": {
+      "content": "__th__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256110": {
+      "content": "__ti__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256111": {
+      "content": "__tl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256112": {
+      "content": "__tn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256113": {
+      "content": "__tr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256114": {
+      "content": "__uk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256115": {
+      "content": "__umb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256116": {
+      "content": "__ur__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256117": {
+      "content": "__uz__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256118": {
+      "content": "__vi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256119": {
+      "content": "__wo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256120": {
+      "content": "__xh__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256121": {
+      "content": "__yi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256122": {
+      "content": "__yo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256123": {
+      "content": "__zh__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256124": {
+      "content": "__zu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "__af__",
+    "__am__",
+    "__ar__",
+    "__as__",
+    "__ast__",
+    "__ay__",
+    "__az__",
+    "__ba__",
+    "__be__",
+    "__bg__",
+    "__bn__",
+    "__br__",
+    "__bs__",
+    "__ca__",
+    "__ceb__",
+    "__cjk__",
+    "__cs__",
+    "__cy__",
+    "__da__",
+    "__de__",
+    "__dyu__",
+    "__el__",
+    "__en__",
+    "__es__",
+    "__et__",
+    "__fa__",
+    "__ff__",
+    "__fi__",
+    "__fr__",
+    "__fy__",
+    "__ga__",
+    "__gd__",
+    "__gl__",
+    "__gu__",
+    "__ha__",
+    "__he__",
+    "__hi__",
+    "__hr__",
+    "__ht__",
+    "__hu__",
+    "__hy__",
+    "__id__",
+    "__ig__",
+    "__ilo__",
+    "__is__",
+    "__it__",
+    "__ja__",
+    "__jv__",
+    "__ka__",
+    "__kac__",
+    "__kam__",
+    "__kea__",
+    "__kg__",
+    "__kk__",
+    "__km__",
+    "__kmb__",
+    "__kmr__",
+    "__kn__",
+    "__ko__",
+    "__ku__",
+    "__ky__",
+    "__lb__",
+    "__lg__",
+    "__ln__",
+    "__lo__",
+    "__lt__",
+    "__luo__",
+    "__lv__",
+    "__mg__",
+    "__mi__",
+    "__mk__",
+    "__ml__",
+    "__mn__",
+    "__mr__",
+    "__ms__",
+    "__mt__",
+    "__my__",
+    "__ne__",
+    "__nl__",
+    "__no__",
+    "__ns__",
+    "__ny__",
+    "__oc__",
+    "__om__",
+    "__or__",
+    "__pa__",
+    "__pl__",
+    "__ps__",
+    "__pt__",
+    "__qu__",
+    "__ro__",
+    "__ru__",
+    "__sd__",
+    "__shn__",
+    "__si__",
+    "__sk__",
+    "__sl__",
+    "__sn__",
+    "__so__",
+    "__sq__",
+    "__sr__",
+    "__ss__",
+    "__su__",
+    "__sv__",
+    "__sw__",
+    "__ta__",
+    "__te__",
+    "__tg__",
+    "__th__",
+    "__ti__",
+    "__tl__",
+    "__tn__",
+    "__tr__",
+    "__uk__",
+    "__umb__",
+    "__ur__",
+    "__uz__",
+    "__vi__",
+    "__wo__",
+    "__xh__",
+    "__yi__",
+    "__yo__",
+    "__zh__",
+    "__zu__"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "language_codes": "m2m100",
+  "model_max_length": 1024,
+  "num_madeup_words": 0,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "src_lang": "en",
+  "tgt_lang": null,
+  "tokenizer_class": "M2M100Tokenizer",
+  "unk_token": "<unk>"
+}

model_files/vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff