Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Clémentine
commited on
Commit
•
bde3a6f
1
Parent(s):
d084b26
fix tokenizer detection
Browse files
src/submission/check_validity.py
CHANGED
@@ -42,7 +42,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
|
|
42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
43 |
if test_tokenizer:
|
44 |
tokenizer_config = get_tokenizer_config(model_name)
|
45 |
-
if tokenizer_config
|
46 |
tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
|
47 |
else:
|
48 |
tokenizer_class_candidate = config.tokenizer_class
|
|
|
42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
43 |
if test_tokenizer:
|
44 |
tokenizer_config = get_tokenizer_config(model_name)
|
45 |
+
if tokenizer_config not in [None, {}]:
|
46 |
tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
|
47 |
else:
|
48 |
tokenizer_class_candidate = config.tokenizer_class
|