Merge pull request #36 from embeddings-benchmark/rmvdups

Update
embeddings-benchmark · Sep 16, 2024 · 0ba2f99 · 0ba2f99
2 parents 96c3b59 + 6f34120
commit 0ba2f99
Show file tree

Hide file tree

Showing 97 changed files with 14,192 additions and 12,822 deletions.
diff --git a/EXTERNAL_MODEL_RESULTS.json b/EXTERNAL_MODEL_RESULTS.json
diff --git a/all_data_tasks/0/default.jsonl b/all_data_tasks/0/default.jsonl
diff --git a/all_data_tasks/1/default.jsonl b/all_data_tasks/1/default.jsonl
diff --git a/all_data_tasks/10/default.jsonl b/all_data_tasks/10/default.jsonl
diff --git a/all_data_tasks/11/default.jsonl b/all_data_tasks/11/default.jsonl
diff --git a/all_data_tasks/12/default.jsonl b/all_data_tasks/12/default.jsonl
diff --git a/all_data_tasks/13/default.jsonl b/all_data_tasks/13/default.jsonl
diff --git a/all_data_tasks/14/default.jsonl b/all_data_tasks/14/default.jsonl
diff --git a/all_data_tasks/15/default.jsonl b/all_data_tasks/15/default.jsonl
diff --git a/all_data_tasks/16/default.jsonl b/all_data_tasks/16/default.jsonl
diff --git a/all_data_tasks/17/default.jsonl b/all_data_tasks/17/default.jsonl
diff --git a/all_data_tasks/18/default.jsonl b/all_data_tasks/18/default.jsonl
diff --git a/all_data_tasks/19/default.jsonl b/all_data_tasks/19/default.jsonl
diff --git a/all_data_tasks/2/default.jsonl b/all_data_tasks/2/default.jsonl
diff --git a/all_data_tasks/20/default.jsonl b/all_data_tasks/20/default.jsonl
diff --git a/all_data_tasks/21/default.jsonl b/all_data_tasks/21/default.jsonl
diff --git a/all_data_tasks/22/default.jsonl b/all_data_tasks/22/default.jsonl
diff --git a/all_data_tasks/23/default.jsonl b/all_data_tasks/23/default.jsonl
diff --git a/all_data_tasks/24/default.jsonl b/all_data_tasks/24/default.jsonl
diff --git a/all_data_tasks/25/default.jsonl b/all_data_tasks/25/default.jsonl
@@ -1,8 +1,8 @@
 {"index":7,"Rank":1,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/deepset\/gbert-large\">gbert-large<\/a>","Model Size (Million Parameters)":337,"Memory Usage (GB, fp32)":1.26,"Embedding Dimensions":1024,"Max Tokens":512,"Average":32.34,"BlurbsClusteringP2P":39.3,"BlurbsClusteringS2S":13.38,"TenKGnadClusteringP2P":41.69,"TenKGnadClusteringS2S":34.97}
 {"index":17,"Rank":2,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/sentence-t5-xxl\">sentence-t5-xxl<\/a>","Model Size (Million Parameters)":4865,"Memory Usage (GB, fp32)":18.12,"Embedding Dimensions":768,"Max Tokens":512,"Average":29.74,"BlurbsClusteringP2P":39.91,"BlurbsClusteringS2S":15.94,"TenKGnadClusteringP2P":43.43,"TenKGnadClusteringS2S":19.69}
-{"index":1,"Rank":3,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/MAY-A\/jina-embeddings-v2-base-de-Q5_K_M-GGUF\">jina-embeddings-v2-base-de-Q5_K_M-GGUF<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Embedding Dimensions":"","Max Tokens":"","Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
-{"index":11,"Rank":4,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/jinaai\/jina-embeddings-v2-base-de\">jina-embeddings-v2-base-de<\/a>","Model Size (Million Parameters)":161,"Memory Usage (GB, fp32)":0.6,"Embedding Dimensions":768,"Max Tokens":8192,"Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
-{"index":5,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/chris-code\/jina-embeddings-v2-base-de-Q8_0-GGUF\">jina-embeddings-v2-base-de-Q8_0-GGUF<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Embedding Dimensions":"","Max Tokens":"","Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
+{"index":11,"Rank":3,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/jinaai\/jina-embeddings-v2-base-de\">jina-embeddings-v2-base-de<\/a>","Model Size (Million Parameters)":161,"Memory Usage (GB, fp32)":0.6,"Embedding Dimensions":768,"Max Tokens":8192,"Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
+{"index":5,"Rank":4,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/chris-code\/jina-embeddings-v2-base-de-Q8_0-GGUF\">jina-embeddings-v2-base-de-Q8_0-GGUF<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Embedding Dimensions":"","Max Tokens":"","Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
+{"index":1,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/MAY-A\/jina-embeddings-v2-base-de-Q5_K_M-GGUF\">jina-embeddings-v2-base-de-Q5_K_M-GGUF<\/a>","Model Size (Million Parameters)":"","Memory Usage (GB, fp32)":"","Embedding Dimensions":"","Max Tokens":"","Average":29.55,"BlurbsClusteringP2P":35.49,"BlurbsClusteringS2S":16.17,"TenKGnadClusteringP2P":42.84,"TenKGnadClusteringS2S":23.69}
 {"index":4,"Rank":6,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/aari1995\/German_Semantic_STS_V2\">German_Semantic_STS_V2<\/a>","Model Size (Million Parameters)":336,"Memory Usage (GB, fp32)":1.25,"Embedding Dimensions":1024,"Max Tokens":512,"Average":29.5,"BlurbsClusteringP2P":38.68,"BlurbsClusteringS2S":17.62,"TenKGnadClusteringP2P":38.0,"TenKGnadClusteringS2S":23.71}
 {"index":16,"Rank":7,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/sentence-t5-large\">sentence-t5-large<\/a>","Model Size (Million Parameters)":168,"Memory Usage (GB, fp32)":0.63,"Embedding Dimensions":768,"Max Tokens":512,"Average":27.49,"BlurbsClusteringP2P":35.33,"BlurbsClusteringS2S":13.27,"TenKGnadClusteringP2P":44.11,"TenKGnadClusteringS2S":17.26}
 {"index":14,"Rank":8,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/paraphrase-multilingual-mpnet-base-v2\">paraphrase-multilingual-mpnet-base-v2<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Embedding Dimensions":768,"Max Tokens":514,"Average":27.04,"BlurbsClusteringP2P":34.38,"BlurbsClusteringS2S":15.81,"TenKGnadClusteringP2P":35.96,"TenKGnadClusteringS2S":22.0}