run dinov2 and some laion models

embeddings-benchmark · Nov 9, 2024 · 2021987 · 2021987
1 parent b5e9fef
commit 2021987
Show file tree

Hide file tree

Showing 13 changed files with 295 additions and 5 deletions.
diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 71.94576096534729,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.3903069466882068,
+        "f1": 0.6257925811318822,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.5316062533656477,
+        "main_score": 0.3903069466882068,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.3905492730210016,
+            "f1": 0.6235333142876379,
+            "lrap": 0.5233687847783186
+          },
+          {
+            "accuracy": 0.401453957996769,
+            "f1": 0.6273491281205146,
+            "lrap": 0.5416105726081543
+          },
+          {
+            "accuracy": 0.37621163166397414,
+            "f1": 0.6161039070570788,
+            "lrap": 0.5324994951534777
+          },
+          {
+            "accuracy": 0.38126009693053314,
+            "f1": 0.6219616890190912,
+            "lrap": 0.5204900376952104
+          },
+          {
+            "accuracy": 0.4020597738287561,
+            "f1": 0.6400148671750886,
+            "lrap": 0.540062376593077
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json
@@ -1 +1 @@
-{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}
diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json
@@ -0,0 +1 @@
+{"name": "facebook/dinov2-giant", "revision": "611a9d42f2335e0f921f1e313ad3c1b7178d206d", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}
diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 146.90911436080933,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.35315024232633274,
+        "f1": 0.5817890400372185,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.49049121567043397,
+        "main_score": 0.35315024232633274,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.3667205169628433,
+            "f1": 0.591090519122089,
+            "lrap": 0.5039753859271227
+          },
+          {
+            "accuracy": 0.34995961227786754,
+            "f1": 0.5618180857706963,
+            "lrap": 0.48571452611739013
+          },
+          {
+            "accuracy": 0.34693053311793215,
+            "f1": 0.5774757520432393,
+            "lrap": 0.49124652216837067
+          },
+          {
+            "accuracy": 0.33582390953150243,
+            "f1": 0.5718289254309215,
+            "lrap": 0.4711502647639497
+          },
+          {
+            "accuracy": 0.36631663974151857,
+            "f1": 0.606731917819147,
+            "lrap": 0.5003693793753363
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json
@@ -1 +1 @@
-{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}
diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 52.176159620285034,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.4340064620355412,
+        "f1": 0.6759098897017933,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.5801337843295732,
+        "main_score": 0.4340064620355412,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.4184168012924071,
+            "f1": 0.6581607635477135,
+            "lrap": 0.563152373900564
+          },
+          {
+            "accuracy": 0.44810177705977383,
+            "f1": 0.6892652132242753,
+            "lrap": 0.5981864790881464
+          },
+          {
+            "accuracy": 0.44426494345718903,
+            "f1": 0.6797744937645096,
+            "lrap": 0.5992208535272061
+          },
+          {
+            "accuracy": 0.43073505654281097,
+            "f1": 0.6706176542382969,
+            "lrap": 0.5713583737210639
+          },
+          {
+            "accuracy": 0.42851373182552505,
+            "f1": 0.6817313237341718,
+            "lrap": 0.5687508414108857
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json
@@ -1 +1 @@
-{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}
diff --git a/...CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json b/...CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 39.63397932052612,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.5000807754442649,
+        "f1": 0.6763180027107614,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.6410416666666808,
+        "main_score": 0.5000807754442649,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.5038368336025848,
+            "f1": 0.6876380960919419,
+            "lrap": 0.6470522572249287
+          },
+          {
+            "accuracy": 0.5232229402261712,
+            "f1": 0.6850605274296695,
+            "lrap": 0.6585363938251828
+          },
+          {
+            "accuracy": 0.4888933764135703,
+            "f1": 0.6592473172415618,
+            "lrap": 0.6350399950637378
+          },
+          {
+            "accuracy": 0.502827140549273,
+            "f1": 0.6767015784006876,
+            "lrap": 0.6400778024591767
+          },
+          {
+            "accuracy": 0.48162358642972536,
+            "f1": 0.6729424943899456,
+            "lrap": 0.6245018847603783
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/...P-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json b/...P-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json
@@ -0,0 +1 @@
+{"name": "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", "revision": "f0e2ffa09cbadab3db6a261ec1ec56407ce42912", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"}
diff --git a/...on__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json b/...on__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 106.33361673355103,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.5382875605815831,
+        "f1": 0.7099573953664177,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.6770555667743827,
+        "main_score": 0.5382875605815831,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.5484652665589661,
+            "f1": 0.7264512368883388,
+            "lrap": 0.6877928109854764
+          },
+          {
+            "accuracy": 0.5581583198707593,
+            "f1": 0.7220847968939508,
+            "lrap": 0.6972356847962836
+          },
+          {
+            "accuracy": 0.5282714054927302,
+            "f1": 0.6913016268300175,
+            "lrap": 0.6676300821217165
+          },
+          {
+            "accuracy": 0.5401857835218093,
+            "f1": 0.7000594523316099,
+            "lrap": 0.6753413323460928
+          },
+          {
+            "accuracy": 0.5163570274636511,
+            "f1": 0.7098898638881714,
+            "lrap": 0.6572779236223446
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/..._CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json b/..._CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json
@@ -1 +1 @@
-{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"}
+{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"}
diff --git a/...on__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json b/...on__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json
@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 139.95830011367798,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.5426898222940226,
+        "f1": 0.7116762127648723,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.6792292676359876,
+        "main_score": 0.5426898222940226,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.5609854604200323,
+            "f1": 0.7310080393154441,
+            "lrap": 0.6944724914737191
+          },
+          {
+            "accuracy": 0.559168012924071,
+            "f1": 0.7125996315013976,
+            "lrap": 0.6941942649434737
+          },
+          {
+            "accuracy": 0.5220113085621971,
+            "f1": 0.6999211686407317,
+            "lrap": 0.6659638081134593
+          },
+          {
+            "accuracy": 0.5387722132471728,
+            "f1": 0.702612150503481,
+            "lrap": 0.6734136600251451
+          },
+          {
+            "accuracy": 0.5325121163166397,
+            "f1": 0.7122400738633065,
+            "lrap": 0.6681021136241404
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}
diff --git a/..._CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json b/..._CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json
@@ -1 +1 @@
-{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"}
+{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"}