tianyuz commited on
Commit
8a10e98
1 Parent(s): 531d636

first commit

Browse files
README.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: ja
3
+ thumbnail: https://github.com/rinnakk/japanese-pretrained-models/blob/master/rinna.png
4
+ tags:
5
+ - ja
6
+ - japanese
7
+ - gpt
8
+ - text-generation
9
+ - lm
10
+ - nlp
11
+ license: mit
12
+ datasets:
13
+ - cc100
14
+ - wikipedia
15
+ widget:
16
+ - text: "西田幾多郎は、"
17
+ ---
18
+
19
+ # japanese-gpt-1b
20
+
21
+ ![rinna-icon](./rinna.png)
22
+
23
+ This repository provides a 1.3B-parameter Japanese GPT model. The model was trained by [rinna Co., Ltd.](https://corp.rinna.co.jp/)
24
+
25
+ # How to use the model
26
+
27
+ *NOTE:* Use `T5Tokenizer` to initiate the tokenizer.
28
+
29
+ ~~~~
30
+ import torch
31
+ from transformers import T5Tokenizer, AutoModelForCausalLM
32
+
33
+ tokenizer = T5Tokenizer.from_pretrained("rinna/japanese-gpt-1b")
34
+ model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b")
35
+
36
+ if torch.cuda.is_available():
37
+ model = model.to("cuda")
38
+
39
+ text = "西田幾多郎は、"
40
+ token_ids = tokenizer.encode(text, add_special_tokens=False, return_tensors="pt")
41
+
42
+ with torch.no_grad():
43
+ output_ids = model.generate(
44
+ token_ids.to(model.device),
45
+ max_length=100,
46
+ min_length=100,
47
+ do_sample=True,
48
+ top_k=500,
49
+ top_p=0.95,
50
+ pad_token_id=tokenizer.pad_token_id,
51
+ bos_token_id=tokenizer.bos_token_id,
52
+ eos_token_id=tokenizer.eos_token_id,
53
+ bad_word_ids=[[tokenizer.unk_token_id]]
54
+ )
55
+
56
+ output = tokenizer.decode(output_ids.tolist()[0])
57
+ print(output)
58
+ ~~~~
59
+
60
+ # Model architecture
61
+ A 24-layer, 2048-hidden-size transformer-based language model.
62
+
63
+ # Training
64
+ The model was trained on [Japanese C4](https://huggingface.co/datasets/allenai/c4), [Japanese CC-100](http://data.statmt.org/cc-100/ja.txt.xz) and [Japanese Wikipedia](https://dumps.wikimedia.org/other/cirrussearch) to optimize a traditional language modelling objective. It reaches around 14 perplexity on a chosen validation set from the same data.
65
+ # Tokenization
66
+ The model uses a [sentencepiece](https://github.com/google/sentencepiece)-based tokenizer. The vocabulary was first trained on a selected subset from the training data using the official sentencepiece training script, and then augmented with emojis and symbols.
67
+ # Licenese
68
+ [The MIT license](https://opensource.org/licenses/MIT)
config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_fast",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 2,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 3,
10
+ "gradient_checkpointing": false,
11
+ "initializer_range": 0.02,
12
+ "layer_norm_epsilon": 1e-05,
13
+ "model_type": "gpt2",
14
+ "n_ctx": 1024,
15
+ "n_embd": 2048,
16
+ "n_head": 16,
17
+ "n_inner": 8192,
18
+ "n_layer": 24,
19
+ "n_positions": 1024,
20
+ "reorder_and_upcast_attn": false,
21
+ "resid_pdrop": 0.1,
22
+ "scale_attn_by_inverse_layer_idx": false,
23
+ "scale_attn_weights": true,
24
+ "use_cache": true,
25
+ "vocab_size": 44928
26
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28a4d618d4665790bc0fd941326f8fbd27fa1f5eebbb406c4000dda34653fcab
3
+ size 2655859801
rinna.png ADDED
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9dbbd4ddbe43941051ed35fd44ff0d9d1c00ed345f7fd4d1969df174110f0609
3
+ size 1044749
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "[PAD]", "extra_ids": 0, "additional_special_tokens": [], "sp_model_kwargs": {}, "bos_token": "<s>", "cls_token": "[CLS]", "sep_token": "[SEP]", "mask_token": "[MASK]", "do_lower_case": false, "tokenizer_class": "T5Tokenizer"}