Automated Leaderboard Update

embeddings-benchmark · Aug 26, 2024 · dce43c1 · dce43c1
1 parent 47b7dda
commit dce43c1
Show file tree

Hide file tree

Showing 61 changed files with 4,827 additions and 4,765 deletions.
diff --git a/all_data_tasks/0/default.jsonl b/all_data_tasks/0/default.jsonl
diff --git a/all_data_tasks/1/default.jsonl b/all_data_tasks/1/default.jsonl
diff --git a/all_data_tasks/10/default.jsonl b/all_data_tasks/10/default.jsonl
diff --git a/all_data_tasks/11/default.jsonl b/all_data_tasks/11/default.jsonl
diff --git a/all_data_tasks/12/default.jsonl b/all_data_tasks/12/default.jsonl
diff --git a/all_data_tasks/13/default.jsonl b/all_data_tasks/13/default.jsonl
diff --git a/all_data_tasks/15/default.jsonl b/all_data_tasks/15/default.jsonl
@@ -2,13 +2,13 @@
 {"index":19,"Rank":2,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/chcaa\/dfm-encoder-large-v1\">dfm-encoder-large-v1<\/a>","Model Size (Million Parameters)":355,"Memory Usage (GB, fp32)":1.32,"Average":59.16,"AngryTweetsClassification":53.8,"DKHateClassification":60.09,"DanishPoliticalCommentsClassification":36.6,"LccSentimentClassification":57.33,"MassiveIntentClassification (da)":60.55,"MassiveScenarioClassification (da)":64.16,"NordicLangClassification":77.68,"ScalaDaClassification":63.08}
 {"index":15,"Rank":3,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/NbAiLab\/nb-bert-large\">nb-bert-large<\/a>","Model Size (Million Parameters)":355,"Memory Usage (GB, fp32)":1.32,"Average":58.9,"AngryTweetsClassification":52.14,"DKHateClassification":62.13,"DanishPoliticalCommentsClassification":35.04,"LccSentimentClassification":56.27,"MassiveIntentClassification (da)":57.03,"MassiveScenarioClassification (da)":60.43,"NordicLangClassification":85.27,"ScalaDaClassification":62.85}
 {"index":25,"Rank":4,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-base\">multilingual-e5-base<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Average":58.61,"AngryTweetsClassification":54.65,"DKHateClassification":63.53,"DanishPoliticalCommentsClassification":36.69,"LccSentimentClassification":59.67,"MassiveIntentClassification (da)":60.16,"MassiveScenarioClassification (da)":67.46,"NordicLangClassification":75.94,"ScalaDaClassification":50.79}
-{"index":49,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/vesteinn\/DanskBERT\">DanskBERT<\/a>","Model Size (Million Parameters)":125,"Memory Usage (GB, fp32)":0.47,"Average":58.31,"AngryTweetsClassification":54.28,"DKHateClassification":59.3,"DanishPoliticalCommentsClassification":39.81,"LccSentimentClassification":58.0,"MassiveIntentClassification (da)":54.68,"MassiveScenarioClassification (da)":59.56,"NordicLangClassification":74.25,"ScalaDaClassification":66.59}
+{"index":50,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/vesteinn\/DanskBERT\">DanskBERT<\/a>","Model Size (Million Parameters)":125,"Memory Usage (GB, fp32)":0.47,"Average":58.31,"AngryTweetsClassification":54.28,"DKHateClassification":59.3,"DanishPoliticalCommentsClassification":39.81,"LccSentimentClassification":58.0,"MassiveIntentClassification (da)":54.68,"MassiveScenarioClassification (da)":59.56,"NordicLangClassification":74.25,"ScalaDaClassification":66.59}
 {"index":14,"Rank":6,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/NbAiLab\/nb-bert-base\">nb-bert-base<\/a>","Model Size (Million Parameters)":179,"Memory Usage (GB, fp32)":0.67,"Average":57.68,"AngryTweetsClassification":52.14,"DKHateClassification":61.73,"DanishPoliticalCommentsClassification":34.84,"LccSentimentClassification":51.4,"MassiveIntentClassification (da)":56.69,"MassiveScenarioClassification (da)":61.93,"NordicLangClassification":84.69,"ScalaDaClassification":57.99}
 {"index":32,"Rank":7,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/ltg\/norbert3-base\">norbert3-base<\/a>","Model Size (Million Parameters)":131,"Memory Usage (GB, fp32)":0.49,"Average":56.34,"AngryTweetsClassification":52.48,"DKHateClassification":58.78,"DanishPoliticalCommentsClassification":34.14,"LccSentimentClassification":54.07,"MassiveIntentClassification (da)":53.16,"MassiveScenarioClassification (da)":57.17,"NordicLangClassification":82.67,"ScalaDaClassification":58.25}
 {"index":28,"Rank":8,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-small\">multilingual-e5-small<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Average":56.12,"AngryTweetsClassification":53.57,"DKHateClassification":60.73,"DanishPoliticalCommentsClassification":34.38,"LccSentimentClassification":57.87,"MassiveIntentClassification (da)":54.63,"MassiveScenarioClassification (da)":62.34,"NordicLangClassification":75.15,"ScalaDaClassification":50.3}
 {"index":20,"Rank":9,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/facebook\/SONAR\">SONAR<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Average":55.03,"AngryTweetsClassification":49.62,"DKHateClassification":69.97,"DanishPoliticalCommentsClassification":37.59,"LccSentimentClassification":54.27,"MassiveIntentClassification (da)":62.03,"MassiveScenarioClassification (da)":67.76,"NordicLangClassification":48.4,"ScalaDaClassification":50.63}
 {"index":33,"Rank":10,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/ltg\/norbert3-large\">norbert3-large<\/a>","Model Size (Million Parameters)":368,"Memory Usage (GB, fp32)":1.37,"Average":54.21,"AngryTweetsClassification":49.04,"DKHateClassification":62.71,"DanishPoliticalCommentsClassification":33.53,"LccSentimentClassification":46.93,"MassiveIntentClassification (da)":45.98,"MassiveScenarioClassification (da)":50.51,"NordicLangClassification":84.25,"ScalaDaClassification":60.72}
-{"index":50,"Rank":11,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/xlm-roberta-base\">xlm-roberta-base<\/a>","Model Size (Million Parameters)":279,"Memory Usage (GB, fp32)":1.04,"Average":52.14,"AngryTweetsClassification":52.41,"DKHateClassification":56.78,"DanishPoliticalCommentsClassification":34.03,"LccSentimentClassification":52.27,"MassiveIntentClassification (da)":41.06,"MassiveScenarioClassification (da)":43.91,"NordicLangClassification":79.39,"ScalaDaClassification":57.3}
+{"index":51,"Rank":11,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/xlm-roberta-base\">xlm-roberta-base<\/a>","Model Size (Million Parameters)":279,"Memory Usage (GB, fp32)":1.04,"Average":52.14,"AngryTweetsClassification":52.41,"DKHateClassification":56.78,"DanishPoliticalCommentsClassification":34.03,"LccSentimentClassification":52.27,"MassiveIntentClassification (da)":41.06,"MassiveScenarioClassification (da)":43.91,"NordicLangClassification":79.39,"ScalaDaClassification":57.3}
 {"index":22,"Rank":12,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/e5-large\">e5-large<\/a>","Model Size (Million Parameters)":335,"Memory Usage (GB, fp32)":1.25,"Average":47.39,"AngryTweetsClassification":46.14,"DKHateClassification":58.72,"DanishPoliticalCommentsClassification":28.67,"LccSentimentClassification":42.13,"MassiveIntentClassification (da)":42.29,"MassiveScenarioClassification (da)":52.95,"NordicLangClassification":58.3,"ScalaDaClassification":49.9}
 {"index":21,"Rank":13,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/e5-base\">e5-base<\/a>","Model Size (Million Parameters)":110,"Memory Usage (GB, fp32)":0.41,"Average":47.02,"AngryTweetsClassification":45.06,"DKHateClassification":58.51,"DanishPoliticalCommentsClassification":28.43,"LccSentimentClassification":37.47,"MassiveIntentClassification (da)":44.25,"MassiveScenarioClassification (da)":52.99,"NordicLangClassification":59.34,"ScalaDaClassification":50.08}
 {"index":8,"Rank":14,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/KBLab\/sentence-bert-swedish-cased\">sentence-bert-swedish-cased<\/a>","Model Size (Million Parameters)":125,"Memory Usage (GB, fp32)":0.47,"Average":46.67,"AngryTweetsClassification":44.46,"DKHateClassification":59.36,"DanishPoliticalCommentsClassification":28.32,"LccSentimentClassification":47.2,"MassiveIntentClassification (da)":42.84,"MassiveScenarioClassification (da)":49.64,"NordicLangClassification":51.45,"ScalaDaClassification":50.12}
@@ -49,3 +49,4 @@
 {"index":46,"Rank":49,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/sentence-t5-xl\">sentence-t5-xl<\/a>","Model Size (Million Parameters)":1240,"Memory Usage (GB, fp32)":4.62,"Average":"","AngryTweetsClassification":"","DKHateClassification":"","DanishPoliticalCommentsClassification":"","LccSentimentClassification":"","MassiveIntentClassification (da)":47.69,"MassiveScenarioClassification (da)":55.79,"NordicLangClassification":"","ScalaDaClassification":""}
 {"index":47,"Rank":50,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sergeyzh\/rubert-tiny-turbo\">rubert-tiny-turbo<\/a>","Model Size (Million Parameters)":29,"Memory Usage (GB, fp32)":0.11,"Average":"","AngryTweetsClassification":"","DKHateClassification":"","DanishPoliticalCommentsClassification":"","LccSentimentClassification":"","MassiveIntentClassification (da)":32.25,"MassiveScenarioClassification (da)":41.14,"NordicLangClassification":"","ScalaDaClassification":""}
 {"index":48,"Rank":51,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/shibing624\/text2vec-base-multilingual\">text2vec-base-multilingual<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Average":"","AngryTweetsClassification":"","DKHateClassification":"","DanishPoliticalCommentsClassification":"","LccSentimentClassification":"","MassiveIntentClassification (da)":56.74,"MassiveScenarioClassification (da)":63.07,"NordicLangClassification":"","ScalaDaClassification":""}
+{"index":49,"Rank":52,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/soichisumi\/multilingual-e5-large-Q8_0-GGUF\">multilingual-e5-large-Q8_0-GGUF<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Average":"","AngryTweetsClassification":"","DKHateClassification":"","DanishPoliticalCommentsClassification":"","LccSentimentClassification":"","MassiveIntentClassification (da)":69.46,"MassiveScenarioClassification (da)":75.06,"NordicLangClassification":"","ScalaDaClassification":""}