Update README.md
Browse files
README.md
CHANGED
@@ -25,13 +25,11 @@ This repository provides a 1.3B-parameter Japanese GPT model. The model was trai
|
|
25 |
|
26 |
# How to use the model
|
27 |
|
28 |
-
*NOTE:* Use `T5Tokenizer` to initiate the tokenizer.
|
29 |
-
|
30 |
~~~~
|
31 |
import torch
|
32 |
-
from transformers import
|
33 |
|
34 |
-
tokenizer =
|
35 |
model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b")
|
36 |
|
37 |
if torch.cuda.is_available():
|
@@ -51,7 +49,7 @@ with torch.no_grad():
|
|
51 |
pad_token_id=tokenizer.pad_token_id,
|
52 |
bos_token_id=tokenizer.bos_token_id,
|
53 |
eos_token_id=tokenizer.eos_token_id,
|
54 |
-
|
55 |
)
|
56 |
|
57 |
output = tokenizer.decode(output_ids.tolist()[0])
|
|
|
25 |
|
26 |
# How to use the model
|
27 |
|
|
|
|
|
28 |
~~~~
|
29 |
import torch
|
30 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM
|
31 |
|
32 |
+
tokenizer = AutoTokenizer.from_pretrained("rinna/japanese-gpt-1b", use_fast=False)
|
33 |
model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b")
|
34 |
|
35 |
if torch.cuda.is_available():
|
|
|
49 |
pad_token_id=tokenizer.pad_token_id,
|
50 |
bos_token_id=tokenizer.bos_token_id,
|
51 |
eos_token_id=tokenizer.eos_token_id,
|
52 |
+
bad_words_ids=[[tokenizer.unk_token_id]]
|
53 |
)
|
54 |
|
55 |
output = tokenizer.decode(output_ids.tolist()[0])
|