Clémentine commited on
Commit
bffc763
·
1 Parent(s): 5c7f7fc

updated caching to include size of models

Browse files
src/display_models/get_model_metadata.py CHANGED
@@ -23,8 +23,13 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
23
  try:
24
  with open("model_info_cache.pkl", "rb") as f:
25
  model_info_cache = pickle.load(f)
26
- except EOFError:
27
  model_info_cache = {}
 
 
 
 
 
28
 
29
  for model_data in tqdm(leaderboard_data):
30
  model_name = model_data["model_name_for_query"]
@@ -39,16 +44,21 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
39
  print("Repo not found!", model_name)
40
  model_data[AutoEvalColumn.license.name] = None
41
  model_data[AutoEvalColumn.likes.name] = None
42
- model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
43
- continue
 
44
 
45
  model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
46
  model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
47
- model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
 
 
48
 
49
  # save cache to disk in pickle format
50
  with open("model_info_cache.pkl", "wb") as f:
51
  pickle.dump(model_info_cache, f)
 
 
52
 
53
 
54
  def get_model_license(model_info):
 
23
  try:
24
  with open("model_info_cache.pkl", "rb") as f:
25
  model_info_cache = pickle.load(f)
26
+ except (EOFError, FileNotFoundError):
27
  model_info_cache = {}
28
+ try:
29
+ with open("model_size_cache.pkl", "rb") as f:
30
+ model_size_cache = pickle.load(f)
31
+ except (EOFError, FileNotFoundError):
32
+ model_size_cache = {}
33
 
34
  for model_data in tqdm(leaderboard_data):
35
  model_name = model_data["model_name_for_query"]
 
44
  print("Repo not found!", model_name)
45
  model_data[AutoEvalColumn.license.name] = None
46
  model_data[AutoEvalColumn.likes.name] = None
47
+ if model_name not in model_size_cache:
48
+ model_size_cache[model_name] = get_model_size(model_name, None)
49
+ model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
50
 
51
  model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
52
  model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
53
+ if model_name not in model_size_cache:
54
+ model_size_cache[model_name] = get_model_size(model_name, model_info)
55
+ model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
56
 
57
  # save cache to disk in pickle format
58
  with open("model_info_cache.pkl", "wb") as f:
59
  pickle.dump(model_info_cache, f)
60
+ with open("model_size_cache.pkl", "wb") as f:
61
+ pickle.dump(model_size_cache, f)
62
 
63
 
64
  def get_model_license(model_info):