init repo

README.md

@@ -1,3 +1,8 @@
 ---
 license: apache-2.0
 ---
+
+Shami Tokenizer
+===
+
+This is the tokenizer of Shami Model.

requirements.txt

@@ -0,0 +1,3 @@
+transformers==4.29.2
+datasets==2.12.0
+apache-beam[gcp]

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_tokenizer.py

@@ -0,0 +1,28 @@
+import json
+
+from transformers import AutoTokenizer
+old_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+import os
+from datasets import load_dataset
+
+langs = ["en", "ja", "ko", "zh-cn", "zh-tw"]
+raw_datasets = [
+    load_dataset("wiki40b", lang, beam_runner='DirectRunner')
+    for lang in langs
+]
+
+total_line = 0
+for training_dataset in raw_datasets:
+    for line in training_dataset["train"]:
+        total_line += 1
+
+def training_dataset_iterator():
+    for training_dataset in raw_datasets:
+        for line in training_dataset["train"]:
+            yield line['text']
+
+# tokenizer.train(training_files, trainer)
+tokenizer = old_tokenizer.train_new_from_iterator(training_dataset_iterator(), 102000, total_line)
+
+tokenizer.save_pretrained("tokenizer-shami")