Spaces:
Running
Running
Clémentine
commited on
Commit
·
bffc763
1
Parent(s):
5c7f7fc
updated caching to include size of models
Browse files
src/display_models/get_model_metadata.py
CHANGED
@@ -23,8 +23,13 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
|
|
23 |
try:
|
24 |
with open("model_info_cache.pkl", "rb") as f:
|
25 |
model_info_cache = pickle.load(f)
|
26 |
-
except EOFError:
|
27 |
model_info_cache = {}
|
|
|
|
|
|
|
|
|
|
|
28 |
|
29 |
for model_data in tqdm(leaderboard_data):
|
30 |
model_name = model_data["model_name_for_query"]
|
@@ -39,16 +44,21 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
|
|
39 |
print("Repo not found!", model_name)
|
40 |
model_data[AutoEvalColumn.license.name] = None
|
41 |
model_data[AutoEvalColumn.likes.name] = None
|
42 |
-
|
43 |
-
|
|
|
44 |
|
45 |
model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
|
46 |
model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
|
47 |
-
|
|
|
|
|
48 |
|
49 |
# save cache to disk in pickle format
|
50 |
with open("model_info_cache.pkl", "wb") as f:
|
51 |
pickle.dump(model_info_cache, f)
|
|
|
|
|
52 |
|
53 |
|
54 |
def get_model_license(model_info):
|
|
|
23 |
try:
|
24 |
with open("model_info_cache.pkl", "rb") as f:
|
25 |
model_info_cache = pickle.load(f)
|
26 |
+
except (EOFError, FileNotFoundError):
|
27 |
model_info_cache = {}
|
28 |
+
try:
|
29 |
+
with open("model_size_cache.pkl", "rb") as f:
|
30 |
+
model_size_cache = pickle.load(f)
|
31 |
+
except (EOFError, FileNotFoundError):
|
32 |
+
model_size_cache = {}
|
33 |
|
34 |
for model_data in tqdm(leaderboard_data):
|
35 |
model_name = model_data["model_name_for_query"]
|
|
|
44 |
print("Repo not found!", model_name)
|
45 |
model_data[AutoEvalColumn.license.name] = None
|
46 |
model_data[AutoEvalColumn.likes.name] = None
|
47 |
+
if model_name not in model_size_cache:
|
48 |
+
model_size_cache[model_name] = get_model_size(model_name, None)
|
49 |
+
model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
|
50 |
|
51 |
model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
|
52 |
model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
|
53 |
+
if model_name not in model_size_cache:
|
54 |
+
model_size_cache[model_name] = get_model_size(model_name, model_info)
|
55 |
+
model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
|
56 |
|
57 |
# save cache to disk in pickle format
|
58 |
with open("model_info_cache.pkl", "wb") as f:
|
59 |
pickle.dump(model_info_cache, f)
|
60 |
+
with open("model_size_cache.pkl", "wb") as f:
|
61 |
+
pickle.dump(model_size_cache, f)
|
62 |
|
63 |
|
64 |
def get_model_license(model_info):
|