Skip to content

Commit

Permalink
Load "full" data to alsao get filenames
Browse files Browse the repository at this point in the history
  • Loading branch information
tomaarsen committed Jul 29, 2024
1 parent 19c9748 commit 046fe24
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 6 deletions.
9 changes: 4 additions & 5 deletions refresh.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ def filter_metric_fetched(name: str, metric: str, expected_metrics) -> bool:


def get_dim_seq_size(model):
filenames = [sib.rfilename for sib in model.siblings]
siblings = model.siblings or []
filenames = [sib.rfilename for sib in siblings]
dim, seq = "", ""
for filename in filenames:
if re.match("\d+_Pooling/config.json", filename):
Expand Down Expand Up @@ -302,7 +303,7 @@ def get_mteb_data(
external_model_results = json.load(f)

api = API
models = list(api.list_models(filter="mteb"))
models = list(api.list_models(filter="mteb", full=True))
# Legacy names changes; Also fetch the old results & merge later
if "MLSUMClusteringP2P (fr)" in datasets:
datasets.append("MLSUMClusteringP2P")
Expand Down Expand Up @@ -429,9 +430,7 @@ def get_mteb_data(
if add_emb_dim:
# The except clause triggers on gated repos, we can use external metadata for those
try:
MODEL_INFOS[model.modelId]["dim_seq_size"] = list(
get_dim_seq_size(model)
)
MODEL_INFOS[model.modelId]["dim_seq_size"] = list(get_dim_seq_size(model))
except:
name_without_org = model.modelId.split("/")[-1]
# EXTERNAL_MODEL_TO_SIZE[name_without_org] refers to millions of parameters, so for memory usage
Expand Down
2 changes: 1 addition & 1 deletion utils/model_size.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def get_model_parameters_memory(model_info: ModelInfo):
try:
safetensors = get_safetensors_metadata(model_info.id)
except Exception as e:
print(traceback.format_exc())
pass
else:
num_parameters = sum(safetensors.parameter_count.values())
return round(num_parameters / 1e6), round(num_parameters * 4 / 1024**3, 2)
Expand Down

0 comments on commit 046fe24

Please sign in to comment.