Update README.md
Browse filesTrailing space breaks tokenization
README.md
CHANGED
@@ -100,7 +100,7 @@ from hf_olmo import OLMoForCausalLM, OLMoTokenizerFast
|
|
100 |
|
101 |
olmo = OLMoForCausalLM.from_pretrained("allenai/OLMo-7B")
|
102 |
tokenizer = OLMoTokenizerFast.from_pretrained("allenai/OLMo-7B")
|
103 |
-
message = ["Language modeling is
|
104 |
inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
|
105 |
# optional verifying cuda
|
106 |
# inputs = {k: v.to('cuda') for k,v in inputs.items()}
|
|
|
100 |
|
101 |
olmo = OLMoForCausalLM.from_pretrained("allenai/OLMo-7B")
|
102 |
tokenizer = OLMoTokenizerFast.from_pretrained("allenai/OLMo-7B")
|
103 |
+
message = ["Language modeling is"]
|
104 |
inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
|
105 |
# optional verifying cuda
|
106 |
# inputs = {k: v.to('cuda') for k,v in inputs.items()}
|