diff --git a/ai-sage__Giga-Embeddings-instruct/external/AmazonCounterfactualClassification.json b/ai-sage__Giga-Embeddings-instruct/external/AmazonCounterfactualClassification.json new file mode 100644 index 000000000..71deec3ec --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/AmazonCounterfactualClassification.json @@ -0,0 +1,179 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "evaluation_time": 199.27316522598267, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.9453523238380809, + "ap": 0.6242264840836734, + "ap_weighted": 0.6242264840836734, + "f1": 0.8713103677336654, + "f1_weighted": 0.9485637995412656, + "hf_subset": "en-ext", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9453523238380809, + "scores_per_experiment": [ + { + "accuracy": 0.972263868065967, + "ap": 0.765857076600446, + "ap_weighted": 0.765857076600446, + "f1": 0.9259479304038374, + "f1_weighted": 0.9723077694286801 + }, + { + "accuracy": 0.9385307346326837, + "ap": 0.5929303585430976, + "ap_weighted": 0.5929303585430976, + "f1": 0.8589546542333717, + "f1_weighted": 0.9428192659116885 + }, + { + "accuracy": 0.9437781109445277, + "ap": 0.6172761239156941, + "ap_weighted": 0.6172761239156941, + "f1": 0.8693023055308442, + "f1_weighted": 0.9474020577819363 + }, + { + "accuracy": 0.9370314842578711, + "ap": 0.5767602868053913, + "ap_weighted": 0.5767602868053913, + "f1": 0.8532322539070483, + "f1_weighted": 0.9410219237983866 + }, + { + "accuracy": 0.9250374812593704, + "ap": 0.5405204784616745, + "ap_weighted": 0.5405204784616745, + "f1": 0.8347701149425288, + "f1_weighted": 0.9314459149735478 + }, + { + "accuracy": 0.9302848575712144, + "ap": 0.5615245170474433, + "ap_weighted": 0.5615245170474433, + "f1": 0.8444314868804664, + "f1_weighted": 0.9359160070110716 + }, + { + "accuracy": 0.9512743628185907, + "ap": 0.6404631060625905, + "ap_weighted": 0.6404631060625905, + "f1": 0.8803714541538878, + "f1_weighted": 0.9532764897429103 + }, + { + "accuracy": 0.9685157421289355, + "ap": 0.7364087610974286, + "ap_weighted": 0.7364087610974286, + "f1": 0.9156738207760151, + "f1_weighted": 0.9685157421289355 + }, + { + "accuracy": 0.9355322338830585, + "ap": 0.5785197882519725, + "ap_weighted": 0.5785197882519725, + "f1": 0.8528341774035477, + "f1_weighted": 0.940163325045911 + }, + { + "accuracy": 0.9512743628185907, + "ap": 0.632004344050996, + "ap_weighted": 0.632004344050996, + "f1": 0.8775854791051075, + "f1_weighted": 0.9527694995895889 + } + ] + }, + { + "accuracy": 0.903134328358209, + "ap": 0.6342364739316405, + "ap_weighted": 0.6342364739316405, + "f1": 0.8554214552412622, + "f1_weighted": 0.9059539168268289, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.903134328358209, + "scores_per_experiment": [ + { + "accuracy": 0.9029850746268657, + "ap": 0.6202534220934586, + "ap_weighted": 0.6202534220934586, + "f1": 0.8496715556491393, + "f1_weighted": 0.9041876352053108 + }, + { + "accuracy": 0.908955223880597, + "ap": 0.6532598866715797, + "ap_weighted": 0.6532598866715797, + "f1": 0.865443686849565, + "f1_weighted": 0.9120387186308277 + }, + { + "accuracy": 0.9014925373134328, + "ap": 0.625611353500422, + "ap_weighted": 0.625611353500422, + "f1": 0.8525528169014085, + "f1_weighted": 0.9042818478032374 + }, + { + "accuracy": 0.9074626865671642, + "ap": 0.6477146043981483, + "ap_weighted": 0.6477146043981483, + "f1": 0.862893430502746, + "f1_weighted": 0.9104961961422293 + }, + { + "accuracy": 0.9149253731343283, + "ap": 0.6667297102274885, + "ap_weighted": 0.6667297102274885, + "f1": 0.8716643311523998, + "f1_weighted": 0.9170383803262476 + }, + { + "accuracy": 0.9, + "ap": 0.6121060646664538, + "ap_weighted": 0.6121060646664538, + "f1": 0.8458929386764438, + "f1_weighted": 0.9014991880971261 + }, + { + "accuracy": 0.9194029850746268, + "ap": 0.6748907174818423, + "ap_weighted": 0.6748907174818423, + "f1": 0.874766355140187, + "f1_weighted": 0.9202957176733156 + }, + { + "accuracy": 0.9283582089552239, + "ap": 0.7029731593764759, + "ap_weighted": 0.7029731593764759, + "f1": 0.8861334957300061, + "f1_weighted": 0.9283582089552239 + }, + { + "accuracy": 0.8820895522388059, + "ap": 0.5850775414601058, + "ap_weighted": 0.5850775414601058, + "f1": 0.8315463176018663, + "f1_weighted": 0.887736071639663 + }, + { + "accuracy": 0.8656716417910447, + "ap": 0.5537482794404294, + "ap_weighted": 0.5537482794404294, + "f1": 0.8136496242088608, + "f1_weighted": 0.8736072037951069 + } + ] + } + ] + }, + "task_name": "AmazonCounterfactualClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/AmazonPolarityClassification.json b/ai-sage__Giga-Embeddings-instruct/external/AmazonPolarityClassification.json new file mode 100644 index 000000000..8a6dae4bf --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/AmazonPolarityClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", + "evaluation_time": 3150.8407397270203, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.9429605000000001, + "ap": 0.9130887530384255, + "ap_weighted": 0.9130887530384255, + "f1": 0.9429070662237378, + "f1_weighted": 0.9429070662237378, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9429605000000001, + "scores_per_experiment": [ + { + "accuracy": 0.938855, + "ap": 0.9100520593271506, + "ap_weighted": 0.9100520593271506, + "f1": 0.9388533687775609, + "f1_weighted": 0.938853368777561 + }, + { + "accuracy": 0.9225475, + "ap": 0.8737119551432224, + "ap_weighted": 0.8737119551432224, + "f1": 0.9223566196264703, + "f1_weighted": 0.9223566196264703 + }, + { + "accuracy": 0.9533625, + "ap": 0.9361003901692877, + "ap_weighted": 0.9361003901692877, + "f1": 0.9533584941216818, + "f1_weighted": 0.9533584941216817 + }, + { + "accuracy": 0.947655, + "ap": 0.9347141077611154, + "ap_weighted": 0.9347141077611154, + "f1": 0.9476225906595737, + "f1_weighted": 0.9476225906595737 + }, + { + "accuracy": 0.94949, + "ap": 0.9254994242406176, + "ap_weighted": 0.9254994242406176, + "f1": 0.9494894811546877, + "f1_weighted": 0.9494894811546876 + }, + { + "accuracy": 0.943205, + "ap": 0.9064453723569432, + "ap_weighted": 0.9064453723569432, + "f1": 0.9431491435204298, + "f1_weighted": 0.9431491435204298 + }, + { + "accuracy": 0.93975, + "ap": 0.9036596651333859, + "ap_weighted": 0.9036596651333859, + "f1": 0.9397089133713968, + "f1_weighted": 0.9397089133713967 + }, + { + "accuracy": 0.9524525, + "ap": 0.9311988050383735, + "ap_weighted": 0.9311988050383735, + "f1": 0.9524524809783178, + "f1_weighted": 0.9524524809783178 + }, + { + "accuracy": 0.952835, + "ap": 0.9282735812161005, + "ap_weighted": 0.9282735812161005, + "f1": 0.9528320301053308, + "f1_weighted": 0.9528320301053308 + }, + { + "accuracy": 0.9294525, + "ap": 0.881232169998059, + "ap_weighted": 0.881232169998059, + "f1": 0.9292475399219274, + "f1_weighted": 0.9292475399219274 + } + ] + } + ] + }, + "task_name": "AmazonPolarityClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/ArguAna.json b/ai-sage__Giga-Embeddings-instruct/external/ArguAna.json new file mode 100644 index 000000000..e54dfb538 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "evaluation_time": 128.91053009033203, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.53228, + "map_at_1": 0.27596, + "map_at_10": 0.43756, + "map_at_100": 0.44674, + "map_at_1000": 0.44675, + "map_at_20": 0.44511, + "map_at_3": 0.38312, + "map_at_5": 0.41271, + "mrr_at_1": 0.2795163584637269, + "mrr_at_10": 0.43868313802524395, + "mrr_at_100": 0.4479916793634115, + "mrr_at_1000": 0.4480064183243461, + "mrr_at_20": 0.4463636850959653, + "mrr_at_3": 0.383831199620673, + "mrr_at_5": 0.41412991939307736, + "nauc_map_at_1000_diff1": 0.06936710279308449, + "nauc_map_at_1000_max": -0.16426102328143827, + "nauc_map_at_1000_std": -0.18408713623781153, + "nauc_map_at_100_diff1": 0.06936900325690783, + "nauc_map_at_100_max": -0.1642459944881398, + "nauc_map_at_100_std": -0.18410024275192619, + "nauc_map_at_10_diff1": 0.06686089466049945, + "nauc_map_at_10_max": -0.16277854443721235, + "nauc_map_at_10_std": -0.18533367246025184, + "nauc_map_at_1_diff1": 0.10048892770421085, + "nauc_map_at_1_max": -0.18880337740587852, + "nauc_map_at_1_std": -0.18950654138263662, + "nauc_map_at_20_diff1": 0.06896257398324564, + "nauc_map_at_20_max": -0.1628720522758851, + "nauc_map_at_20_std": -0.18463554340157873, + "nauc_map_at_3_diff1": 0.06996349008138944, + "nauc_map_at_3_max": -0.16895326699141894, + "nauc_map_at_3_std": -0.18550696483491105, + "nauc_map_at_5_diff1": 0.06652257808997529, + "nauc_map_at_5_max": -0.16616340120756665, + "nauc_map_at_5_std": -0.18750380766744815, + "nauc_mrr_at_1000_diff1": 0.05675242976111991, + "nauc_mrr_at_1000_max": -0.16992812047837066, + "nauc_mrr_at_1000_std": -0.18329294971328722, + "nauc_mrr_at_100_diff1": 0.056754937777142835, + "nauc_mrr_at_100_max": -0.16991287123334947, + "nauc_mrr_at_100_std": -0.18330604638796044, + "nauc_mrr_at_10_diff1": 0.053927681776353154, + "nauc_mrr_at_10_max": -0.16891663162548254, + "nauc_mrr_at_10_std": -0.18471864534496943, + "nauc_mrr_at_1_diff1": 0.08923777873913467, + "nauc_mrr_at_1_max": -0.18816652686644939, + "nauc_mrr_at_1_std": -0.18819665466571675, + "nauc_mrr_at_20_diff1": 0.056417523389287016, + "nauc_mrr_at_20_max": -0.16851365689901593, + "nauc_mrr_at_20_std": -0.18384362648232547, + "nauc_mrr_at_3_diff1": 0.05524316132813568, + "nauc_mrr_at_3_max": -0.1772356834345999, + "nauc_mrr_at_3_std": -0.18372688451025657, + "nauc_mrr_at_5_diff1": 0.054144051832033246, + "nauc_mrr_at_5_max": -0.17288127460794153, + "nauc_mrr_at_5_std": -0.1871123050851349, + "nauc_ndcg_at_1000_diff1": 0.06487802962417492, + "nauc_ndcg_at_1000_max": -0.1576159401306176, + "nauc_ndcg_at_1000_std": -0.1815838595665605, + "nauc_ndcg_at_100_diff1": 0.0648323468898899, + "nauc_ndcg_at_100_max": -0.15728467477722477, + "nauc_ndcg_at_100_std": -0.1819738421807864, + "nauc_ndcg_at_10_diff1": 0.054234480184110256, + "nauc_ndcg_at_10_max": -0.14673502378215453, + "nauc_ndcg_at_10_std": -0.18837931889895315, + "nauc_ndcg_at_1_diff1": 0.10048892770421085, + "nauc_ndcg_at_1_max": -0.18880337740587852, + "nauc_ndcg_at_1_std": -0.18950654138263662, + "nauc_ndcg_at_20_diff1": 0.06369954849420038, + "nauc_ndcg_at_20_max": -0.14443991776264714, + "nauc_ndcg_at_20_std": -0.18416264332865837, + "nauc_ndcg_at_3_diff1": 0.06224331563078568, + "nauc_ndcg_at_3_max": -0.16183370694913554, + "nauc_ndcg_at_3_std": -0.18559481650690338, + "nauc_ndcg_at_5_diff1": 0.05659342042143409, + "nauc_ndcg_at_5_max": -0.15510631438356692, + "nauc_ndcg_at_5_std": -0.18909647623269873, + "nauc_precision_at_1000_diff1": -0.45740924328524435, + "nauc_precision_at_1000_max": -0.044367453191845235, + "nauc_precision_at_1000_std": 0.5794428979357973, + "nauc_precision_at_100_diff1": -0.2375197189716444, + "nauc_precision_at_100_max": 0.00510917620494902, + "nauc_precision_at_100_std": 0.1413313021307472, + "nauc_precision_at_10_diff1": -0.022741922400170953, + "nauc_precision_at_10_max": -0.04695134136659869, + "nauc_precision_at_10_std": -0.21566024184206758, + "nauc_precision_at_1_diff1": 0.10048892770421085, + "nauc_precision_at_1_max": -0.18880337740587852, + "nauc_precision_at_1_std": -0.18950654138263662, + "nauc_precision_at_20_diff1": 0.02366832261816588, + "nauc_precision_at_20_max": 0.17078759245976263, + "nauc_precision_at_20_std": -0.1757368482497663, + "nauc_precision_at_3_diff1": 0.04062538060385958, + "nauc_precision_at_3_max": -0.1410949953336873, + "nauc_precision_at_3_std": -0.18626114079282416, + "nauc_precision_at_5_diff1": 0.02425834990396102, + "nauc_precision_at_5_max": -0.11600278541101094, + "nauc_precision_at_5_std": -0.19533267961798942, + "nauc_recall_at_1000_diff1": -0.45740924328527977, + "nauc_recall_at_1000_max": -0.04436745319187756, + "nauc_recall_at_1000_std": 0.579442897935769, + "nauc_recall_at_100_diff1": -0.23751971897160465, + "nauc_recall_at_100_max": 0.005109176204928447, + "nauc_recall_at_100_std": 0.14133130213071957, + "nauc_recall_at_10_diff1": -0.022741922400170526, + "nauc_recall_at_10_max": -0.04695134136659741, + "nauc_recall_at_10_std": -0.21566024184206647, + "nauc_recall_at_1_diff1": 0.10048892770421085, + "nauc_recall_at_1_max": -0.18880337740587852, + "nauc_recall_at_1_std": -0.18950654138263662, + "nauc_recall_at_20_diff1": 0.023668322618168716, + "nauc_recall_at_20_max": 0.17078759245976433, + "nauc_recall_at_20_std": -0.1757368482497646, + "nauc_recall_at_3_diff1": 0.040625380603860055, + "nauc_recall_at_3_max": -0.1410949953336872, + "nauc_recall_at_3_std": -0.18626114079282394, + "nauc_recall_at_5_diff1": 0.02425834990396135, + "nauc_recall_at_5_max": -0.1160027854110106, + "nauc_recall_at_5_std": -0.19533267961798925, + "ndcg_at_1": 0.27596, + "ndcg_at_10": 0.53228, + "ndcg_at_100": 0.56931, + "ndcg_at_1000": 0.56968, + "ndcg_at_20": 0.55921, + "ndcg_at_3": 0.41908, + "ndcg_at_5": 0.47285, + "precision_at_1": 0.27596, + "precision_at_10": 0.08371, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04712, + "precision_at_3": 0.17449, + "precision_at_5": 0.13101, + "recall_at_1": 0.27596, + "recall_at_10": 0.83713, + "recall_at_100": 0.9936, + "recall_at_1000": 0.99644, + "recall_at_20": 0.94239, + "recall_at_3": 0.52347, + "recall_at_5": 0.65505 + } + ] + }, + "task_name": "ArguAna" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/AskUbuntuDupQuestions.json b/ai-sage__Giga-Embeddings-instruct/external/AskUbuntuDupQuestions.json new file mode 100644 index 000000000..279f1a2fc --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "evaluation_time": 28.4865984916687, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5818725222465893, + "map": 0.5818725222465893, + "mrr": 0.6984335839598997, + "nAUC_map_diff1": 0.1911598989756231, + "nAUC_map_max": 0.15695053858587465, + "nAUC_map_std": 0.22147773436080342, + "nAUC_mrr_diff1": 0.2538427130339339, + "nAUC_mrr_max": 0.242962940173052, + "nAUC_mrr_std": 0.20846599304343177 + } + ] + }, + "task_name": "AskUbuntuDupQuestions" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/BIOSSES.json b/ai-sage__Giga-Embeddings-instruct/external/BIOSSES.json new file mode 100644 index 000000000..5a3abf5ee --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "evaluation_time": 17.42747926712036, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8740814966131607, + "cosine_spearman": 0.8421758160533057, + "euclidean_pearson": 0.8668087011664756, + "euclidean_spearman": 0.8421758160533057, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8421758160533057, + "manhattan_pearson": 0.868885717540405, + "manhattan_spearman": 0.8469409848718736, + "pearson": 0.8740814966131607, + "spearman": 0.8421758160533057 + } + ] + }, + "task_name": "BIOSSES" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/Banking77Classification.json b/ai-sage__Giga-Embeddings-instruct/external/Banking77Classification.json new file mode 100644 index 000000000..35d3e4a83 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "evaluation_time": 121.5467758178711, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.8778571428571429, + "f1": 0.8755183393575304, + "f1_weighted": 0.8755183393575307, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8778571428571429, + "scores_per_experiment": [ + { + "accuracy": 0.8762987012987012, + "f1": 0.8739366457035869, + "f1_weighted": 0.8739366457035866 + }, + { + "accuracy": 0.8737012987012988, + "f1": 0.871042814913296, + "f1_weighted": 0.8710428149132963 + }, + { + "accuracy": 0.887012987012987, + "f1": 0.8841164392967462, + "f1_weighted": 0.8841164392967463 + }, + { + "accuracy": 0.8801948051948052, + "f1": 0.8791582793250402, + "f1_weighted": 0.8791582793250403 + }, + { + "accuracy": 0.8714285714285714, + "f1": 0.8686773798364148, + "f1_weighted": 0.8686773798364147 + }, + { + "accuracy": 0.8795454545454545, + "f1": 0.8774171972906563, + "f1_weighted": 0.8774171972906566 + }, + { + "accuracy": 0.8788961038961038, + "f1": 0.8770763555557125, + "f1_weighted": 0.8770763555557127 + }, + { + "accuracy": 0.8746753246753247, + "f1": 0.8715123524921125, + "f1_weighted": 0.8715123524921123 + }, + { + "accuracy": 0.8782467532467533, + "f1": 0.875363373968927, + "f1_weighted": 0.8753633739689269 + }, + { + "accuracy": 0.8785714285714286, + "f1": 0.8768825551928128, + "f1_weighted": 0.8768825551928131 + } + ] + } + ] + }, + "task_name": "Banking77Classification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CEDRClassification.json b/ai-sage__Giga-Embeddings-instruct/external/CEDRClassification.json new file mode 100644 index 000000000..0afce2ec4 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CEDRClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", "evaluation_time": 28.821460485458374, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.596894792773645, "f1": 0.5907371458842751, "hf_subset": "default", "languages": ["rus-Cyrl"], "lrap": 0.8446838469713137, "main_score": 0.596894792773645, "scores_per_experiment": [{"accuracy": 0.5914218916046758, "f1": 0.5952633918007464, "lrap": 0.8353878852284873}, {"accuracy": 0.6371179596174282, "f1": 0.6089772845492801, "lrap": 0.8556057385759889}, {"accuracy": 0.6296790648246546, "f1": 0.5871318433863721, "lrap": 0.8198724760892743}, {"accuracy": 0.6174580233793836, "f1": 0.6081778543559243, "lrap": 0.858342189160474}, {"accuracy": 0.628616365568544, "f1": 0.6009472840247019, "lrap": 0.8454303931987314}, {"accuracy": 0.59992348565356, "f1": 0.6092999118078863, "lrap": 0.8477683315621743}, {"accuracy": 0.5664484590860787, "f1": 0.5733575072650084, "lrap": 0.8386291179596241}, {"accuracy": 0.5823889479277364, "f1": 0.5485643502482982, "lrap": 0.8463868225292309}, {"accuracy": 0.5770754516471839, "f1": 0.6011522448754063, "lrap": 0.8636556854410259}, {"accuracy": 0.5388182784272051, "f1": 0.5744997865291266, "lrap": 0.8357598299681251}]}]}, "task_name": "CEDRClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackAndroidRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackAndroidRetrieval.json new file mode 100644 index 000000000..0c7d5d3c7 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackAndroidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb", + "evaluation_time": 174.20506191253662, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.36264, + "map_at_1": 0.2154, + "map_at_10": 0.30146, + "map_at_100": 0.31733, + "map_at_1000": 0.31909, + "map_at_20": 0.30973, + "map_at_3": 0.27021, + "map_at_5": 0.28444, + "mrr_at_1": 0.2718168812589413, + "mrr_at_10": 0.3602663669187272, + "mrr_at_100": 0.37033436947159926, + "mrr_at_1000": 0.3708950227995111, + "mrr_at_20": 0.3657523894791198, + "mrr_at_3": 0.3307105388650455, + "mrr_at_5": 0.3450882212684788, + "nauc_map_at_1000_diff1": 0.40005919478157476, + "nauc_map_at_1000_max": 0.24149847825797918, + "nauc_map_at_1000_std": 0.032086670255845405, + "nauc_map_at_100_diff1": 0.4000441108717158, + "nauc_map_at_100_max": 0.24055155307330164, + "nauc_map_at_100_std": 0.03220400139532892, + "nauc_map_at_10_diff1": 0.3992034196847358, + "nauc_map_at_10_max": 0.23481367728480562, + "nauc_map_at_10_std": 0.024652443046052703, + "nauc_map_at_1_diff1": 0.4521095152281425, + "nauc_map_at_1_max": 0.23426091924328707, + "nauc_map_at_1_std": -0.027839519807285072, + "nauc_map_at_20_diff1": 0.39765141613442495, + "nauc_map_at_20_max": 0.238640869346293, + "nauc_map_at_20_std": 0.029991581054265748, + "nauc_map_at_3_diff1": 0.406459399697723, + "nauc_map_at_3_max": 0.23680185542835097, + "nauc_map_at_3_std": 0.0029419617901594664, + "nauc_map_at_5_diff1": 0.3993079810788306, + "nauc_map_at_5_max": 0.23524497440705255, + "nauc_map_at_5_std": 0.017962358900605612, + "nauc_mrr_at_1000_diff1": 0.39109444381118774, + "nauc_mrr_at_1000_max": 0.24778123336724733, + "nauc_mrr_at_1000_std": 0.030165840634640846, + "nauc_mrr_at_100_diff1": 0.39107349821363074, + "nauc_mrr_at_100_max": 0.24766616675099815, + "nauc_mrr_at_100_std": 0.030334155163562613, + "nauc_mrr_at_10_diff1": 0.39037151411366816, + "nauc_mrr_at_10_max": 0.24599134492259705, + "nauc_mrr_at_10_std": 0.029789305275262683, + "nauc_mrr_at_1_diff1": 0.4496218824994712, + "nauc_mrr_at_1_max": 0.2559168953889985, + "nauc_mrr_at_1_std": -0.013310616968087602, + "nauc_mrr_at_20_diff1": 0.38909044438924995, + "nauc_mrr_at_20_max": 0.24621874344604244, + "nauc_mrr_at_20_std": 0.029656421549078667, + "nauc_mrr_at_3_diff1": 0.39363648096874354, + "nauc_mrr_at_3_max": 0.24843395616218664, + "nauc_mrr_at_3_std": 0.013819897830014732, + "nauc_mrr_at_5_diff1": 0.38863436520404, + "nauc_mrr_at_5_max": 0.2490556151740949, + "nauc_mrr_at_5_std": 0.027820696970598652, + "nauc_ndcg_at_1000_diff1": 0.38833224287044255, + "nauc_ndcg_at_1000_max": 0.2527156294182745, + "nauc_ndcg_at_1000_std": 0.06973933838132665, + "nauc_ndcg_at_100_diff1": 0.3840151337957579, + "nauc_ndcg_at_100_max": 0.241600176041664, + "nauc_ndcg_at_100_std": 0.07235920148334983, + "nauc_ndcg_at_10_diff1": 0.37503092819367506, + "nauc_ndcg_at_10_max": 0.23024367502884033, + "nauc_ndcg_at_10_std": 0.049955965690241265, + "nauc_ndcg_at_1_diff1": 0.4496218824994712, + "nauc_ndcg_at_1_max": 0.2559168953889985, + "nauc_ndcg_at_1_std": -0.013310616968087602, + "nauc_ndcg_at_20_diff1": 0.36843167364410345, + "nauc_ndcg_at_20_max": 0.2335930351746899, + "nauc_ndcg_at_20_std": 0.05764156579665355, + "nauc_ndcg_at_3_diff1": 0.39005209938080365, + "nauc_ndcg_at_3_max": 0.24336089017831652, + "nauc_ndcg_at_3_std": 0.019558269186682103, + "nauc_ndcg_at_5_diff1": 0.37765417510974536, + "nauc_ndcg_at_5_max": 0.23935465539942227, + "nauc_ndcg_at_5_std": 0.04178781488281662, + "nauc_precision_at_1000_diff1": -0.10372529546949247, + "nauc_precision_at_1000_max": 0.03594993364396436, + "nauc_precision_at_1000_std": -0.03360438126279742, + "nauc_precision_at_100_diff1": 0.03555249109060567, + "nauc_precision_at_100_max": 0.13448701405797095, + "nauc_precision_at_100_std": 0.1262161529298294, + "nauc_precision_at_10_diff1": 0.19337662094452096, + "nauc_precision_at_10_max": 0.22307700535828423, + "nauc_precision_at_10_std": 0.14350516411235187, + "nauc_precision_at_1_diff1": 0.4496218824994712, + "nauc_precision_at_1_max": 0.2559168953889985, + "nauc_precision_at_1_std": -0.013310616968087602, + "nauc_precision_at_20_diff1": 0.12196602490339437, + "nauc_precision_at_20_max": 0.21283209534212133, + "nauc_precision_at_20_std": 0.15740621879710157, + "nauc_precision_at_3_diff1": 0.3136910311849708, + "nauc_precision_at_3_max": 0.25298131418390063, + "nauc_precision_at_3_std": 0.06505279304669079, + "nauc_precision_at_5_diff1": 0.26088097622906997, + "nauc_precision_at_5_max": 0.25166926352778274, + "nauc_precision_at_5_std": 0.12204299883327283, + "nauc_recall_at_1000_diff1": 0.34255882215302047, + "nauc_recall_at_1000_max": 0.27835351170558503, + "nauc_recall_at_1000_std": 0.5956089209650579, + "nauc_recall_at_100_diff1": 0.2850733988410035, + "nauc_recall_at_100_max": 0.164950314804097, + "nauc_recall_at_100_std": 0.23991797968783846, + "nauc_recall_at_10_diff1": 0.2944195661453061, + "nauc_recall_at_10_max": 0.16630540783772904, + "nauc_recall_at_10_std": 0.10542317179779533, + "nauc_recall_at_1_diff1": 0.4521095152281425, + "nauc_recall_at_1_max": 0.23426091924328707, + "nauc_recall_at_1_std": -0.027839519807285072, + "nauc_recall_at_20_diff1": 0.2529147097583405, + "nauc_recall_at_20_max": 0.16973739781631583, + "nauc_recall_at_20_std": 0.1262845697358077, + "nauc_recall_at_3_diff1": 0.3453023294888764, + "nauc_recall_at_3_max": 0.21410528878364002, + "nauc_recall_at_3_std": 0.03333564391257637, + "nauc_recall_at_5_diff1": 0.31549298686623894, + "nauc_recall_at_5_max": 0.20019545741301525, + "nauc_recall_at_5_std": 0.08638769848309126, + "ndcg_at_1": 0.27182, + "ndcg_at_10": 0.36264, + "ndcg_at_100": 0.42935, + "ndcg_at_1000": 0.4564, + "ndcg_at_20": 0.38619, + "ndcg_at_3": 0.30971, + "ndcg_at_5": 0.32799, + "precision_at_1": 0.27182, + "precision_at_10": 0.07396, + "precision_at_100": 0.01375, + "precision_at_1000": 0.00206, + "precision_at_20": 0.04571, + "precision_at_3": 0.15069, + "precision_at_5": 0.10959, + "recall_at_1": 0.2154, + "recall_at_10": 0.48304, + "recall_at_100": 0.7724, + "recall_at_1000": 0.94405, + "recall_at_20": 0.56845, + "recall_at_3": 0.32828, + "recall_at_5": 0.38029 + } + ] + }, + "task_name": "CQADupstackAndroidRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackEnglishRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackEnglishRetrieval.json new file mode 100644 index 000000000..2fe4ba91a --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackEnglishRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0", + "evaluation_time": 232.1940095424652, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.32666, + "map_at_1": 0.19496, + "map_at_10": 0.27708, + "map_at_100": 0.28885, + "map_at_1000": 0.2903, + "map_at_20": 0.28347, + "map_at_3": 0.25188, + "map_at_5": 0.26757, + "mrr_at_1": 0.24140127388535032, + "mrr_at_10": 0.3226827418865631, + "mrr_at_100": 0.33191905671640304, + "mrr_at_1000": 0.33252344963858865, + "mrr_at_20": 0.3283671423953111, + "mrr_at_3": 0.29893842887473426, + "mrr_at_5": 0.31419320594479766, + "nauc_map_at_1000_diff1": 0.41766175261661953, + "nauc_map_at_1000_max": 0.3236105298409472, + "nauc_map_at_1000_std": 0.036495511496519153, + "nauc_map_at_100_diff1": 0.4178018349925877, + "nauc_map_at_100_max": 0.32281085114700364, + "nauc_map_at_100_std": 0.035129991248899074, + "nauc_map_at_10_diff1": 0.4198486116566451, + "nauc_map_at_10_max": 0.32047237898567715, + "nauc_map_at_10_std": 0.027440607363035923, + "nauc_map_at_1_diff1": 0.4714494297512505, + "nauc_map_at_1_max": 0.3072990453372588, + "nauc_map_at_1_std": -0.014748123251677639, + "nauc_map_at_20_diff1": 0.4187357825935804, + "nauc_map_at_20_max": 0.32074571884360614, + "nauc_map_at_20_std": 0.030650478802649508, + "nauc_map_at_3_diff1": 0.4344660277541162, + "nauc_map_at_3_max": 0.315736965905546, + "nauc_map_at_3_std": 0.012023760907565172, + "nauc_map_at_5_diff1": 0.42633442437395874, + "nauc_map_at_5_max": 0.3167816525700815, + "nauc_map_at_5_std": 0.020332027700373904, + "nauc_mrr_at_1000_diff1": 0.40219481670580187, + "nauc_mrr_at_1000_max": 0.33427537717907363, + "nauc_mrr_at_1000_std": 0.05933854401399415, + "nauc_mrr_at_100_diff1": 0.40207921855075407, + "nauc_mrr_at_100_max": 0.33409626556544186, + "nauc_mrr_at_100_std": 0.059276614649538394, + "nauc_mrr_at_10_diff1": 0.4024913874148852, + "nauc_mrr_at_10_max": 0.3344670144717516, + "nauc_mrr_at_10_std": 0.05676393545620746, + "nauc_mrr_at_1_diff1": 0.46383192519266575, + "nauc_mrr_at_1_max": 0.3474359913713124, + "nauc_mrr_at_1_std": 0.03082081961113807, + "nauc_mrr_at_20_diff1": 0.4012631408589766, + "nauc_mrr_at_20_max": 0.33372139587730265, + "nauc_mrr_at_20_std": 0.058614253543251026, + "nauc_mrr_at_3_diff1": 0.4176392677686872, + "nauc_mrr_at_3_max": 0.3375783375631955, + "nauc_mrr_at_3_std": 0.05242263033695972, + "nauc_mrr_at_5_diff1": 0.4066128066488706, + "nauc_mrr_at_5_max": 0.33223646792379563, + "nauc_mrr_at_5_std": 0.054305287334867655, + "nauc_ndcg_at_1000_diff1": 0.3863357657667927, + "nauc_ndcg_at_1000_max": 0.3352315610562187, + "nauc_ndcg_at_1000_std": 0.08432732805029033, + "nauc_ndcg_at_100_diff1": 0.38603550373052853, + "nauc_ndcg_at_100_max": 0.3278553003015518, + "nauc_ndcg_at_100_std": 0.07665142929815671, + "nauc_ndcg_at_10_diff1": 0.3917218005607024, + "nauc_ndcg_at_10_max": 0.3237648122534756, + "nauc_ndcg_at_10_std": 0.05229213836558875, + "nauc_ndcg_at_1_diff1": 0.46383192519266575, + "nauc_ndcg_at_1_max": 0.3474359913713124, + "nauc_ndcg_at_1_std": 0.03082081961113807, + "nauc_ndcg_at_20_diff1": 0.3883973319985223, + "nauc_ndcg_at_20_max": 0.3216751688681974, + "nauc_ndcg_at_20_std": 0.059529585154962, + "nauc_ndcg_at_3_diff1": 0.41445258776543653, + "nauc_ndcg_at_3_max": 0.32427076262318477, + "nauc_ndcg_at_3_std": 0.035522432042425216, + "nauc_ndcg_at_5_diff1": 0.4020334324643417, + "nauc_ndcg_at_5_max": 0.31866704090360176, + "nauc_ndcg_at_5_std": 0.04129311453116481, + "nauc_precision_at_1000_diff1": -0.027079160999067653, + "nauc_precision_at_1000_max": 0.1396843551636217, + "nauc_precision_at_1000_std": 0.2267619361153759, + "nauc_precision_at_100_diff1": 0.05367108735367731, + "nauc_precision_at_100_max": 0.2326131673320251, + "nauc_precision_at_100_std": 0.2650328168346979, + "nauc_precision_at_10_diff1": 0.21861845852630704, + "nauc_precision_at_10_max": 0.3303504705148769, + "nauc_precision_at_10_std": 0.17695093243551263, + "nauc_precision_at_1_diff1": 0.46383192519266575, + "nauc_precision_at_1_max": 0.3474359913713124, + "nauc_precision_at_1_std": 0.03082081961113807, + "nauc_precision_at_20_diff1": 0.16086225605250912, + "nauc_precision_at_20_max": 0.29111406274685686, + "nauc_precision_at_20_std": 0.20160476272916578, + "nauc_precision_at_3_diff1": 0.3487770156593762, + "nauc_precision_at_3_max": 0.34365988003728853, + "nauc_precision_at_3_std": 0.09713422411448411, + "nauc_precision_at_5_diff1": 0.28556995540691216, + "nauc_precision_at_5_max": 0.32417757045903506, + "nauc_precision_at_5_std": 0.12526037082673244, + "nauc_recall_at_1000_diff1": 0.22518923633684243, + "nauc_recall_at_1000_max": 0.34777567979920276, + "nauc_recall_at_1000_std": 0.3096602342569516, + "nauc_recall_at_100_diff1": 0.26977344899465067, + "nauc_recall_at_100_max": 0.2925577591297841, + "nauc_recall_at_100_std": 0.18533293276892132, + "nauc_recall_at_10_diff1": 0.31833116167561254, + "nauc_recall_at_10_max": 0.2953270517235027, + "nauc_recall_at_10_std": 0.07576504321049565, + "nauc_recall_at_1_diff1": 0.4714494297512505, + "nauc_recall_at_1_max": 0.3072990453372588, + "nauc_recall_at_1_std": -0.014748123251677639, + "nauc_recall_at_20_diff1": 0.29522302969072467, + "nauc_recall_at_20_max": 0.27881678480167693, + "nauc_recall_at_20_std": 0.100707944266602, + "nauc_recall_at_3_diff1": 0.3857308910390624, + "nauc_recall_at_3_max": 0.2997474141447162, + "nauc_recall_at_3_std": 0.03239683467317649, + "nauc_recall_at_5_diff1": 0.35398282044000434, + "nauc_recall_at_5_max": 0.29029908154639783, + "nauc_recall_at_5_std": 0.05034024931108196, + "ndcg_at_1": 0.2414, + "ndcg_at_10": 0.32666, + "ndcg_at_100": 0.37734, + "ndcg_at_1000": 0.40511, + "ndcg_at_20": 0.34628, + "ndcg_at_3": 0.28509, + "ndcg_at_5": 0.30813, + "precision_at_1": 0.2414, + "precision_at_10": 0.06248, + "precision_at_100": 0.01113, + "precision_at_1000": 0.00169, + "precision_at_20": 0.03803, + "precision_at_3": 0.13737, + "precision_at_5": 0.10242, + "recall_at_1": 0.19496, + "recall_at_10": 0.42402, + "recall_at_100": 0.64252, + "recall_at_1000": 0.82322, + "recall_at_20": 0.4965, + "recall_at_3": 0.30927, + "recall_at_5": 0.36829 + } + ] + }, + "task_name": "CQADupstackEnglishRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGamingRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGamingRetrieval.json new file mode 100644 index 000000000..493ea8ff0 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "evaluation_time": 264.64156198501587, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.39403, + "map_at_1": 0.23953, + "map_at_10": 0.33771, + "map_at_100": 0.35095, + "map_at_1000": 0.35184, + "map_at_20": 0.34531, + "map_at_3": 0.30705, + "map_at_5": 0.32468, + "mrr_at_1": 0.27711598746081506, + "mrr_at_10": 0.3688045479424789, + "mrr_at_100": 0.37979904815602633, + "mrr_at_1000": 0.3802816011291392, + "mrr_at_20": 0.37550760856437426, + "mrr_at_3": 0.34190177638453473, + "mrr_at_5": 0.35707419017763836, + "nauc_map_at_1000_diff1": 0.3600981920501309, + "nauc_map_at_1000_max": 0.22095524568873015, + "nauc_map_at_1000_std": -0.028665621594465306, + "nauc_map_at_100_diff1": 0.36002011911243975, + "nauc_map_at_100_max": 0.22104565543592558, + "nauc_map_at_100_std": -0.02867602396981025, + "nauc_map_at_10_diff1": 0.3578951669531971, + "nauc_map_at_10_max": 0.21454593639617814, + "nauc_map_at_10_std": -0.03697485560054045, + "nauc_map_at_1_diff1": 0.40534778283633016, + "nauc_map_at_1_max": 0.18467906438990472, + "nauc_map_at_1_std": -0.0722165065662934, + "nauc_map_at_20_diff1": 0.36010838525304834, + "nauc_map_at_20_max": 0.21885046883123593, + "nauc_map_at_20_std": -0.03210375937430325, + "nauc_map_at_3_diff1": 0.37418022634214515, + "nauc_map_at_3_max": 0.21073669021006833, + "nauc_map_at_3_std": -0.04679404664686779, + "nauc_map_at_5_diff1": 0.366795123593822, + "nauc_map_at_5_max": 0.21389015358073685, + "nauc_map_at_5_std": -0.04325648408069168, + "nauc_mrr_at_1000_diff1": 0.34956116409457827, + "nauc_mrr_at_1000_max": 0.23928491774707167, + "nauc_mrr_at_1000_std": -0.01338791971396386, + "nauc_mrr_at_100_diff1": 0.3492919352840362, + "nauc_mrr_at_100_max": 0.23939943348709594, + "nauc_mrr_at_100_std": -0.013082650041533628, + "nauc_mrr_at_10_diff1": 0.3469031733385043, + "nauc_mrr_at_10_max": 0.2359274582738218, + "nauc_mrr_at_10_std": -0.018289688836101457, + "nauc_mrr_at_1_diff1": 0.39347341432081323, + "nauc_mrr_at_1_max": 0.2150394761714022, + "nauc_mrr_at_1_std": -0.05387089032695643, + "nauc_mrr_at_20_diff1": 0.34913032503137814, + "nauc_mrr_at_20_max": 0.23847491555315675, + "nauc_mrr_at_20_std": -0.014304205162891498, + "nauc_mrr_at_3_diff1": 0.3618259864668598, + "nauc_mrr_at_3_max": 0.23887811503907033, + "nauc_mrr_at_3_std": -0.02464439729797598, + "nauc_mrr_at_5_diff1": 0.35334375318458494, + "nauc_mrr_at_5_max": 0.23765452586249292, + "nauc_mrr_at_5_std": -0.02185185539542626, + "nauc_ndcg_at_1000_diff1": 0.34094187426792627, + "nauc_ndcg_at_1000_max": 0.2455238871231702, + "nauc_ndcg_at_1000_std": 0.013417861971730751, + "nauc_ndcg_at_100_diff1": 0.3350605572479613, + "nauc_ndcg_at_100_max": 0.24788837094333716, + "nauc_ndcg_at_100_std": 0.0194905635328888, + "nauc_ndcg_at_10_diff1": 0.3299617129249672, + "nauc_ndcg_at_10_max": 0.22373041946409547, + "nauc_ndcg_at_10_std": -0.016123251442731485, + "nauc_ndcg_at_1_diff1": 0.39347341432081323, + "nauc_ndcg_at_1_max": 0.2150394761714022, + "nauc_ndcg_at_1_std": -0.05387089032695643, + "nauc_ndcg_at_20_diff1": 0.33761532851653864, + "nauc_ndcg_at_20_max": 0.23662812360354787, + "nauc_ndcg_at_20_std": 0.001314652340010358, + "nauc_ndcg_at_3_diff1": 0.3621562886201887, + "nauc_ndcg_at_3_max": 0.22237288102218208, + "nauc_ndcg_at_3_std": -0.03375324547297835, + "nauc_ndcg_at_5_diff1": 0.3486998025049706, + "nauc_ndcg_at_5_max": 0.22375617254550176, + "nauc_ndcg_at_5_std": -0.028765947297571596, + "nauc_precision_at_1000_diff1": -0.054771331359408426, + "nauc_precision_at_1000_max": 0.11714063664418657, + "nauc_precision_at_1000_std": 0.14803511361321767, + "nauc_precision_at_100_diff1": 0.01190308506686791, + "nauc_precision_at_100_max": 0.20879498285480025, + "nauc_precision_at_100_std": 0.19864715556174226, + "nauc_precision_at_10_diff1": 0.16690689283041116, + "nauc_precision_at_10_max": 0.23615626182492824, + "nauc_precision_at_10_std": 0.06960499622103876, + "nauc_precision_at_1_diff1": 0.39347341432081323, + "nauc_precision_at_1_max": 0.2150394761714022, + "nauc_precision_at_1_std": -0.05387089032695643, + "nauc_precision_at_20_diff1": 0.1394137013326293, + "nauc_precision_at_20_max": 0.250983395906178, + "nauc_precision_at_20_std": 0.12863365250801706, + "nauc_precision_at_3_diff1": 0.2970706108250014, + "nauc_precision_at_3_max": 0.2597857880896327, + "nauc_precision_at_3_std": 0.004671634667547243, + "nauc_precision_at_5_diff1": 0.2426712316402683, + "nauc_precision_at_5_max": 0.24994467132277265, + "nauc_precision_at_5_std": 0.021020033045283954, + "nauc_recall_at_1000_diff1": 0.21036527285628887, + "nauc_recall_at_1000_max": 0.4635945439743593, + "nauc_recall_at_1000_std": 0.4768561830236934, + "nauc_recall_at_100_diff1": 0.19685774418991592, + "nauc_recall_at_100_max": 0.34825810808385366, + "nauc_recall_at_100_std": 0.2758546312971761, + "nauc_recall_at_10_diff1": 0.23749182280335082, + "nauc_recall_at_10_max": 0.21230541488908958, + "nauc_recall_at_10_std": 0.025379471535492675, + "nauc_recall_at_1_diff1": 0.40534778283633016, + "nauc_recall_at_1_max": 0.18467906438990472, + "nauc_recall_at_1_std": -0.0722165065662934, + "nauc_recall_at_20_diff1": 0.2573985930226769, + "nauc_recall_at_20_max": 0.25552244146602354, + "nauc_recall_at_20_std": 0.09719380851403539, + "nauc_recall_at_3_diff1": 0.3344976234422277, + "nauc_recall_at_3_max": 0.2210543969450477, + "nauc_recall_at_3_std": -0.017497643333208013, + "nauc_recall_at_5_diff1": 0.2995702802348527, + "nauc_recall_at_5_max": 0.2190857504616491, + "nauc_recall_at_5_std": -0.0066343753107045485, + "ndcg_at_1": 0.27712, + "ndcg_at_10": 0.39403, + "ndcg_at_100": 0.45327, + "ndcg_at_1000": 0.47221, + "ndcg_at_20": 0.41921, + "ndcg_at_3": 0.33657, + "ndcg_at_5": 0.36463, + "precision_at_1": 0.27712, + "precision_at_10": 0.06715, + "precision_at_100": 0.01068, + "precision_at_1000": 0.0013, + "precision_at_20": 0.04034, + "precision_at_3": 0.15361, + "precision_at_5": 0.1106, + "recall_at_1": 0.23953, + "recall_at_10": 0.53554, + "recall_at_100": 0.79397, + "recall_at_1000": 0.92859, + "recall_at_20": 0.62906, + "recall_at_3": 0.38074, + "recall_at_5": 0.44869 + } + ] + }, + "task_name": "CQADupstackGamingRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGisRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGisRetrieval.json new file mode 100644 index 000000000..1a3c047bd --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackGisRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2", + "evaluation_time": 601.0864291191101, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.20723, + "map_at_1": 0.11454, + "map_at_10": 0.16933, + "map_at_100": 0.17931, + "map_at_1000": 0.18066, + "map_at_20": 0.17444, + "map_at_3": 0.14821, + "map_at_5": 0.15847, + "mrr_at_1": 0.1288135593220339, + "mrr_at_10": 0.18568648551699388, + "mrr_at_100": 0.19519728948766965, + "mrr_at_1000": 0.19629818718309142, + "mrr_at_20": 0.1904251586406436, + "mrr_at_3": 0.1638418079096045, + "mrr_at_5": 0.17446327683615817, + "nauc_map_at_1000_diff1": 0.3100971211104956, + "nauc_map_at_1000_max": 0.13850017784244986, + "nauc_map_at_1000_std": 0.08963659587175178, + "nauc_map_at_100_diff1": 0.30974620576122003, + "nauc_map_at_100_max": 0.13853314356539528, + "nauc_map_at_100_std": 0.08954247164461093, + "nauc_map_at_10_diff1": 0.31109975868798434, + "nauc_map_at_10_max": 0.1351348567087792, + "nauc_map_at_10_std": 0.08550888012900434, + "nauc_map_at_1_diff1": 0.3984333952813111, + "nauc_map_at_1_max": 0.16068746301688736, + "nauc_map_at_1_std": 0.04093189268230556, + "nauc_map_at_20_diff1": 0.3085752130769573, + "nauc_map_at_20_max": 0.1363962555746682, + "nauc_map_at_20_std": 0.08709432284644775, + "nauc_map_at_3_diff1": 0.3280973726691313, + "nauc_map_at_3_max": 0.14284720966613204, + "nauc_map_at_3_std": 0.06160852529090238, + "nauc_map_at_5_diff1": 0.31808730696207005, + "nauc_map_at_5_max": 0.13834242310684614, + "nauc_map_at_5_std": 0.06835723601421637, + "nauc_mrr_at_1000_diff1": 0.2975904642335793, + "nauc_mrr_at_1000_max": 0.14951338971704445, + "nauc_mrr_at_1000_std": 0.10113349487019041, + "nauc_mrr_at_100_diff1": 0.2971206355838227, + "nauc_mrr_at_100_max": 0.149588343997006, + "nauc_mrr_at_100_std": 0.10147966222836807, + "nauc_mrr_at_10_diff1": 0.296269898474758, + "nauc_mrr_at_10_max": 0.1474042625656872, + "nauc_mrr_at_10_std": 0.09839974138708314, + "nauc_mrr_at_1_diff1": 0.3838214921179599, + "nauc_mrr_at_1_max": 0.16904444938773946, + "nauc_mrr_at_1_std": 0.054614336143629115, + "nauc_mrr_at_20_diff1": 0.2958274441739926, + "nauc_mrr_at_20_max": 0.14873525526797793, + "nauc_mrr_at_20_std": 0.1002659346294331, + "nauc_mrr_at_3_diff1": 0.31348878321254803, + "nauc_mrr_at_3_max": 0.15483058306281644, + "nauc_mrr_at_3_std": 0.08200409977944424, + "nauc_mrr_at_5_diff1": 0.3016593682015114, + "nauc_mrr_at_5_max": 0.1485982071153109, + "nauc_mrr_at_5_std": 0.08185908135154363, + "nauc_ndcg_at_1000_diff1": 0.2818547695388205, + "nauc_ndcg_at_1000_max": 0.14005327206216434, + "nauc_ndcg_at_1000_std": 0.12589669560088967, + "nauc_ndcg_at_100_diff1": 0.27758324300357984, + "nauc_ndcg_at_100_max": 0.14341133555972088, + "nauc_ndcg_at_100_std": 0.1331250167325408, + "nauc_ndcg_at_10_diff1": 0.27905796760757134, + "nauc_ndcg_at_10_max": 0.1296750126038255, + "nauc_ndcg_at_10_std": 0.11252396272365991, + "nauc_ndcg_at_1_diff1": 0.3838214921179599, + "nauc_ndcg_at_1_max": 0.16904444938773946, + "nauc_ndcg_at_1_std": 0.054614336143629115, + "nauc_ndcg_at_20_diff1": 0.2734987634408996, + "nauc_ndcg_at_20_max": 0.13320914532200256, + "nauc_ndcg_at_20_std": 0.11867410167033551, + "nauc_ndcg_at_3_diff1": 0.3034325923729927, + "nauc_ndcg_at_3_max": 0.14201276650661435, + "nauc_ndcg_at_3_std": 0.0712523611875583, + "nauc_ndcg_at_5_diff1": 0.28801943226027754, + "nauc_ndcg_at_5_max": 0.13338551217810954, + "nauc_ndcg_at_5_std": 0.07801128464664271, + "nauc_precision_at_1000_diff1": 0.020012681522812617, + "nauc_precision_at_1000_max": 0.10612211087722671, + "nauc_precision_at_1000_std": 0.18232225350370507, + "nauc_precision_at_100_diff1": 0.12999552197478134, + "nauc_precision_at_100_max": 0.1633859747515745, + "nauc_precision_at_100_std": 0.2645969466756656, + "nauc_precision_at_10_diff1": 0.18117672151473987, + "nauc_precision_at_10_max": 0.12306841727543809, + "nauc_precision_at_10_std": 0.19224817620570556, + "nauc_precision_at_1_diff1": 0.3838214921179599, + "nauc_precision_at_1_max": 0.16904444938773946, + "nauc_precision_at_1_std": 0.054614336143629115, + "nauc_precision_at_20_diff1": 0.15115529221920224, + "nauc_precision_at_20_max": 0.1289077612414914, + "nauc_precision_at_20_std": 0.20414316524315318, + "nauc_precision_at_3_diff1": 0.2503722914022995, + "nauc_precision_at_3_max": 0.15709378585870357, + "nauc_precision_at_3_std": 0.10010187249834739, + "nauc_precision_at_5_diff1": 0.21525625871418208, + "nauc_precision_at_5_max": 0.13497803642649472, + "nauc_precision_at_5_std": 0.11075155507096765, + "nauc_recall_at_1000_diff1": 0.18280987536015267, + "nauc_recall_at_1000_max": 0.09574683101626953, + "nauc_recall_at_1000_std": 0.23448036040224915, + "nauc_recall_at_100_diff1": 0.20578493216367275, + "nauc_recall_at_100_max": 0.1456962816210217, + "nauc_recall_at_100_std": 0.2341963721799639, + "nauc_recall_at_10_diff1": 0.21583257614269807, + "nauc_recall_at_10_max": 0.10242354988584784, + "nauc_recall_at_10_std": 0.1598063513181825, + "nauc_recall_at_1_diff1": 0.3984333952813111, + "nauc_recall_at_1_max": 0.16068746301688736, + "nauc_recall_at_1_std": 0.04093189268230556, + "nauc_recall_at_20_diff1": 0.20352337875547352, + "nauc_recall_at_20_max": 0.11292402190116818, + "nauc_recall_at_20_std": 0.17604505218199173, + "nauc_recall_at_3_diff1": 0.2604480028663088, + "nauc_recall_at_3_max": 0.1275221283182816, + "nauc_recall_at_3_std": 0.07771755623504321, + "nauc_recall_at_5_diff1": 0.2267971113828535, + "nauc_recall_at_5_max": 0.10998067327387379, + "nauc_recall_at_5_std": 0.08733032345555437, + "ndcg_at_1": 0.12881, + "ndcg_at_10": 0.20723, + "ndcg_at_100": 0.26068, + "ndcg_at_1000": 0.29594, + "ndcg_at_20": 0.22462, + "ndcg_at_3": 0.16351, + "ndcg_at_5": 0.18119, + "precision_at_1": 0.12881, + "precision_at_10": 0.03638, + "precision_at_100": 0.00675, + "precision_at_1000": 0.00102, + "precision_at_20": 0.0222, + "precision_at_3": 0.07269, + "precision_at_5": 0.05379, + "recall_at_1": 0.11454, + "recall_at_10": 0.31119, + "recall_at_100": 0.56649, + "recall_at_1000": 0.83599, + "recall_at_20": 0.37657, + "recall_at_3": 0.19035, + "recall_at_5": 0.23307 + } + ] + }, + "task_name": "CQADupstackGisRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackMathematicaRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackMathematicaRetrieval.json new file mode 100644 index 000000000..f184a1fc6 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de", + "evaluation_time": 397.68456983566284, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.18602, + "map_at_1": 0.08233, + "map_at_10": 0.14263, + "map_at_100": 0.15635, + "map_at_1000": 0.15782, + "map_at_20": 0.14959, + "map_at_3": 0.11889, + "map_at_5": 0.13067, + "mrr_at_1": 0.09950248756218906, + "mrr_at_10": 0.16844300323777928, + "mrr_at_100": 0.18059357585498775, + "mrr_at_1000": 0.18154524917526424, + "mrr_at_20": 0.17474440783148787, + "mrr_at_3": 0.14510779436152563, + "mrr_at_5": 0.15667495854063018, + "nauc_map_at_1000_diff1": 0.1823217746190047, + "nauc_map_at_1000_max": 0.2049022753078953, + "nauc_map_at_1000_std": 0.14846867678528303, + "nauc_map_at_100_diff1": 0.18222951182346728, + "nauc_map_at_100_max": 0.20462428579695047, + "nauc_map_at_100_std": 0.14801451610995042, + "nauc_map_at_10_diff1": 0.1880699816824651, + "nauc_map_at_10_max": 0.20346327683018395, + "nauc_map_at_10_std": 0.14781486965673699, + "nauc_map_at_1_diff1": 0.2490271842269989, + "nauc_map_at_1_max": 0.20605821895768833, + "nauc_map_at_1_std": 0.13654354962612034, + "nauc_map_at_20_diff1": 0.1830322394754611, + "nauc_map_at_20_max": 0.2025406227052277, + "nauc_map_at_20_std": 0.1433873197584387, + "nauc_map_at_3_diff1": 0.20174467160467258, + "nauc_map_at_3_max": 0.1911985380269457, + "nauc_map_at_3_std": 0.12503935147559375, + "nauc_map_at_5_diff1": 0.18687432760432812, + "nauc_map_at_5_max": 0.1987068913715553, + "nauc_map_at_5_std": 0.1410119379199484, + "nauc_mrr_at_1000_diff1": 0.15686926847621882, + "nauc_mrr_at_1000_max": 0.20418385041948672, + "nauc_mrr_at_1000_std": 0.14601678542982033, + "nauc_mrr_at_100_diff1": 0.15649148415618785, + "nauc_mrr_at_100_max": 0.20390471061201454, + "nauc_mrr_at_100_std": 0.14570935730839452, + "nauc_mrr_at_10_diff1": 0.15799150500411854, + "nauc_mrr_at_10_max": 0.20575744858600387, + "nauc_mrr_at_10_std": 0.14768268932423928, + "nauc_mrr_at_1_diff1": 0.23367444138495524, + "nauc_mrr_at_1_max": 0.20695137503965666, + "nauc_mrr_at_1_std": 0.1472157770136156, + "nauc_mrr_at_20_diff1": 0.1569282182722306, + "nauc_mrr_at_20_max": 0.2049672528700865, + "nauc_mrr_at_20_std": 0.14388349301754727, + "nauc_mrr_at_3_diff1": 0.16552667983195965, + "nauc_mrr_at_3_max": 0.1930089479846865, + "nauc_mrr_at_3_std": 0.12419365827975917, + "nauc_mrr_at_5_diff1": 0.152021345488141, + "nauc_mrr_at_5_max": 0.19863155583605863, + "nauc_mrr_at_5_std": 0.13981416058831525, + "nauc_ndcg_at_1000_diff1": 0.15496613143969418, + "nauc_ndcg_at_1000_max": 0.22093806024787851, + "nauc_ndcg_at_1000_std": 0.17766995355844745, + "nauc_ndcg_at_100_diff1": 0.15256710001281315, + "nauc_ndcg_at_100_max": 0.21858707665995042, + "nauc_ndcg_at_100_std": 0.1706357990658541, + "nauc_ndcg_at_10_diff1": 0.16645396625915107, + "nauc_ndcg_at_10_max": 0.21533816489312796, + "nauc_ndcg_at_10_std": 0.1629564024579396, + "nauc_ndcg_at_1_diff1": 0.23367444138495524, + "nauc_ndcg_at_1_max": 0.20695137503965666, + "nauc_ndcg_at_1_std": 0.1472157770136156, + "nauc_ndcg_at_20_diff1": 0.155570017125561, + "nauc_ndcg_at_20_max": 0.21359185072455986, + "nauc_ndcg_at_20_std": 0.15057162550772377, + "nauc_ndcg_at_3_diff1": 0.18148078513804156, + "nauc_ndcg_at_3_max": 0.19413219163942882, + "nauc_ndcg_at_3_std": 0.12206859061209872, + "nauc_ndcg_at_5_diff1": 0.15946387582108482, + "nauc_ndcg_at_5_max": 0.20614438981415711, + "nauc_ndcg_at_5_std": 0.14956938588009588, + "nauc_precision_at_1000_diff1": -0.06338405284332406, + "nauc_precision_at_1000_max": 0.06367574020314141, + "nauc_precision_at_1000_std": 0.032948111384908635, + "nauc_precision_at_100_diff1": 0.025963271076466284, + "nauc_precision_at_100_max": 0.17195718920975406, + "nauc_precision_at_100_std": 0.1344480018562989, + "nauc_precision_at_10_diff1": 0.09894164875058406, + "nauc_precision_at_10_max": 0.22607176912959512, + "nauc_precision_at_10_std": 0.16602991509521992, + "nauc_precision_at_1_diff1": 0.23367444138495524, + "nauc_precision_at_1_max": 0.20695137503965666, + "nauc_precision_at_1_std": 0.1472157770136156, + "nauc_precision_at_20_diff1": 0.07479617555395288, + "nauc_precision_at_20_max": 0.20646048773417566, + "nauc_precision_at_20_std": 0.12587065342832732, + "nauc_precision_at_3_diff1": 0.13549141346398993, + "nauc_precision_at_3_max": 0.19096896253669218, + "nauc_precision_at_3_std": 0.10986362155695349, + "nauc_precision_at_5_diff1": 0.09308854693332992, + "nauc_precision_at_5_max": 0.20614038596547596, + "nauc_precision_at_5_std": 0.14519366227680033, + "nauc_recall_at_1000_diff1": 0.11654281536115599, + "nauc_recall_at_1000_max": 0.32692947390599003, + "nauc_recall_at_1000_std": 0.4465436374659935, + "nauc_recall_at_100_diff1": 0.10060501930914467, + "nauc_recall_at_100_max": 0.23909265280968484, + "nauc_recall_at_100_std": 0.23087966012102398, + "nauc_recall_at_10_diff1": 0.14302911670798507, + "nauc_recall_at_10_max": 0.22858699104110153, + "nauc_recall_at_10_std": 0.18963781625147183, + "nauc_recall_at_1_diff1": 0.2490271842269989, + "nauc_recall_at_1_max": 0.20605821895768833, + "nauc_recall_at_1_std": 0.13654354962612034, + "nauc_recall_at_20_diff1": 0.11411188213614896, + "nauc_recall_at_20_max": 0.22259549408442764, + "nauc_recall_at_20_std": 0.15507984814176626, + "nauc_recall_at_3_diff1": 0.16269091809041347, + "nauc_recall_at_3_max": 0.18870658327598097, + "nauc_recall_at_3_std": 0.11838400434931914, + "nauc_recall_at_5_diff1": 0.12244714380165343, + "nauc_recall_at_5_max": 0.21261392668644533, + "nauc_recall_at_5_std": 0.16651686356883158, + "ndcg_at_1": 0.0995, + "ndcg_at_10": 0.18602, + "ndcg_at_100": 0.25512, + "ndcg_at_1000": 0.2926, + "ndcg_at_20": 0.2102, + "ndcg_at_3": 0.13972, + "ndcg_at_5": 0.15877, + "precision_at_1": 0.0995, + "precision_at_10": 0.03943, + "precision_at_100": 0.00872, + "precision_at_1000": 0.00136, + "precision_at_20": 0.02643, + "precision_at_3": 0.07255, + "precision_at_5": 0.05622, + "recall_at_1": 0.08233, + "recall_at_10": 0.29367, + "recall_at_100": 0.60209, + "recall_at_1000": 0.87415, + "recall_at_20": 0.3819, + "recall_at_3": 0.16519, + "recall_at_5": 0.21319 + } + ] + }, + "task_name": "CQADupstackMathematicaRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackPhysicsRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackPhysicsRetrieval.json new file mode 100644 index 000000000..24061472a --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4", + "evaluation_time": 414.0358958244324, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.32592, + "map_at_1": 0.18758, + "map_at_10": 0.27198, + "map_at_100": 0.28706, + "map_at_1000": 0.28838, + "map_at_20": 0.28033, + "map_at_3": 0.24353, + "map_at_5": 0.25971, + "mrr_at_1": 0.24254090471607315, + "mrr_at_10": 0.32293375803962887, + "mrr_at_100": 0.3333713398298678, + "mrr_at_1000": 0.33409272424477227, + "mrr_at_20": 0.3286529042825282, + "mrr_at_3": 0.2977221687520053, + "mrr_at_5": 0.31158164902149493, + "nauc_map_at_1000_diff1": 0.3543332186007521, + "nauc_map_at_1000_max": 0.28979622893773105, + "nauc_map_at_1000_std": 0.06578956825302608, + "nauc_map_at_100_diff1": 0.35390221586614484, + "nauc_map_at_100_max": 0.2894560731774703, + "nauc_map_at_100_std": 0.06538195339460331, + "nauc_map_at_10_diff1": 0.35677884861017917, + "nauc_map_at_10_max": 0.2857489475080354, + "nauc_map_at_10_std": 0.05454103100616544, + "nauc_map_at_1_diff1": 0.42534580883690243, + "nauc_map_at_1_max": 0.25886547599537935, + "nauc_map_at_1_std": 0.001163937107467496, + "nauc_map_at_20_diff1": 0.3548823910499732, + "nauc_map_at_20_max": 0.2866122249354595, + "nauc_map_at_20_std": 0.06073699959355872, + "nauc_map_at_3_diff1": 0.3701024770409362, + "nauc_map_at_3_max": 0.2796181204492567, + "nauc_map_at_3_std": 0.04451362489929877, + "nauc_map_at_5_diff1": 0.36694282631483194, + "nauc_map_at_5_max": 0.2780209824957949, + "nauc_map_at_5_std": 0.04297801872956301, + "nauc_mrr_at_1000_diff1": 0.34559056308808933, + "nauc_mrr_at_1000_max": 0.32836661998719446, + "nauc_mrr_at_1000_std": 0.11867050560410966, + "nauc_mrr_at_100_diff1": 0.3452327347415353, + "nauc_mrr_at_100_max": 0.3282393565429814, + "nauc_mrr_at_100_std": 0.11866910023146005, + "nauc_mrr_at_10_diff1": 0.34532086131009926, + "nauc_mrr_at_10_max": 0.3285146332978944, + "nauc_mrr_at_10_std": 0.1171169987538628, + "nauc_mrr_at_1_diff1": 0.39335582470379094, + "nauc_mrr_at_1_max": 0.31740439567406037, + "nauc_mrr_at_1_std": 0.08529530277559985, + "nauc_mrr_at_20_diff1": 0.34544134792349557, + "nauc_mrr_at_20_max": 0.3280857217884902, + "nauc_mrr_at_20_std": 0.11817454680274084, + "nauc_mrr_at_3_diff1": 0.35727640697384316, + "nauc_mrr_at_3_max": 0.3304686562750682, + "nauc_mrr_at_3_std": 0.1145610782490127, + "nauc_mrr_at_5_diff1": 0.3533942729546681, + "nauc_mrr_at_5_max": 0.3247716771142299, + "nauc_mrr_at_5_std": 0.10970511640489987, + "nauc_ndcg_at_1000_diff1": 0.32972563023860496, + "nauc_ndcg_at_1000_max": 0.3156124332885808, + "nauc_ndcg_at_1000_std": 0.11320625017933687, + "nauc_ndcg_at_100_diff1": 0.31981617985898936, + "nauc_ndcg_at_100_max": 0.3081744850721845, + "nauc_ndcg_at_100_std": 0.11051314311449151, + "nauc_ndcg_at_10_diff1": 0.32898755384329703, + "nauc_ndcg_at_10_max": 0.2983589745108241, + "nauc_ndcg_at_10_std": 0.0808277274745963, + "nauc_ndcg_at_1_diff1": 0.39335582470379094, + "nauc_ndcg_at_1_max": 0.31740439567406037, + "nauc_ndcg_at_1_std": 0.08529530277559985, + "nauc_ndcg_at_20_diff1": 0.32412206781494796, + "nauc_ndcg_at_20_max": 0.29861450129085315, + "nauc_ndcg_at_20_std": 0.09383718452743128, + "nauc_ndcg_at_3_diff1": 0.3498413605059997, + "nauc_ndcg_at_3_max": 0.3014318309067973, + "nauc_ndcg_at_3_std": 0.07909917756441662, + "nauc_ndcg_at_5_diff1": 0.3492814489705384, + "nauc_ndcg_at_5_max": 0.2916608661742856, + "nauc_ndcg_at_5_std": 0.06694550269553597, + "nauc_precision_at_1000_diff1": -0.03790848810395332, + "nauc_precision_at_1000_max": 0.14802421936399268, + "nauc_precision_at_1000_std": 0.23145894849144555, + "nauc_precision_at_100_diff1": 0.017546038039919793, + "nauc_precision_at_100_max": 0.25662733100762297, + "nauc_precision_at_100_std": 0.31225277020769826, + "nauc_precision_at_10_diff1": 0.15694416287771043, + "nauc_precision_at_10_max": 0.326564497075471, + "nauc_precision_at_10_std": 0.2234022846945397, + "nauc_precision_at_1_diff1": 0.39335582470379094, + "nauc_precision_at_1_max": 0.31740439567406037, + "nauc_precision_at_1_std": 0.08529530277559985, + "nauc_precision_at_20_diff1": 0.10453000063206942, + "nauc_precision_at_20_max": 0.2905806853523417, + "nauc_precision_at_20_std": 0.26277225739049487, + "nauc_precision_at_3_diff1": 0.27272188733861136, + "nauc_precision_at_3_max": 0.34839330622142767, + "nauc_precision_at_3_std": 0.17775294609166153, + "nauc_precision_at_5_diff1": 0.24258907574802577, + "nauc_precision_at_5_max": 0.32119190912429113, + "nauc_precision_at_5_std": 0.16252625454457664, + "nauc_recall_at_1000_diff1": 0.14954086634753433, + "nauc_recall_at_1000_max": 0.4280424989945532, + "nauc_recall_at_1000_std": 0.36798723724134463, + "nauc_recall_at_100_diff1": 0.16048538619104163, + "nauc_recall_at_100_max": 0.27368927292992323, + "nauc_recall_at_100_std": 0.18887258365172094, + "nauc_recall_at_10_diff1": 0.2446647338879468, + "nauc_recall_at_10_max": 0.26603563703221095, + "nauc_recall_at_10_std": 0.08028133219751867, + "nauc_recall_at_1_diff1": 0.42534580883690243, + "nauc_recall_at_1_max": 0.25886547599537935, + "nauc_recall_at_1_std": 0.001163937107467496, + "nauc_recall_at_20_diff1": 0.2240840509718229, + "nauc_recall_at_20_max": 0.25805462126399786, + "nauc_recall_at_20_std": 0.11278990263026085, + "nauc_recall_at_3_diff1": 0.31262823269379075, + "nauc_recall_at_3_max": 0.27044002473477063, + "nauc_recall_at_3_std": 0.058901711913007304, + "nauc_recall_at_5_diff1": 0.3018775263414265, + "nauc_recall_at_5_max": 0.25583478864235903, + "nauc_recall_at_5_std": 0.04569298948312717, + "ndcg_at_1": 0.24254, + "ndcg_at_10": 0.32592, + "ndcg_at_100": 0.39334, + "ndcg_at_1000": 0.42145, + "ndcg_at_20": 0.35195, + "ndcg_at_3": 0.27742, + "ndcg_at_5": 0.30035, + "precision_at_1": 0.24254, + "precision_at_10": 0.06381, + "precision_at_100": 0.01149, + "precision_at_1000": 0.00158, + "precision_at_20": 0.03989, + "precision_at_3": 0.13474, + "precision_at_5": 0.09933, + "recall_at_1": 0.18758, + "recall_at_10": 0.43732, + "recall_at_100": 0.73152, + "recall_at_1000": 0.92088, + "recall_at_20": 0.53056, + "recall_at_3": 0.30348, + "recall_at_5": 0.36248 + } + ] + }, + "task_name": "CQADupstackPhysicsRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackProgrammersRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackProgrammersRetrieval.json new file mode 100644 index 000000000..6669a4990 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32", + "evaluation_time": 356.1611044406891, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.27024, + "map_at_1": 0.14198, + "map_at_10": 0.2175, + "map_at_100": 0.23328, + "map_at_1000": 0.2347, + "map_at_20": 0.22595, + "map_at_3": 0.189, + "map_at_5": 0.20362, + "mrr_at_1": 0.1815068493150685, + "mrr_at_10": 0.26040262375878803, + "mrr_at_100": 0.27198199648239774, + "mrr_at_1000": 0.2727774611735573, + "mrr_at_20": 0.2669194118908377, + "mrr_at_3": 0.23173515981735168, + "mrr_at_5": 0.24686073059360727, + "nauc_map_at_1000_diff1": 0.31053453200715325, + "nauc_map_at_1000_max": 0.29024393074756616, + "nauc_map_at_1000_std": 0.06325913499941616, + "nauc_map_at_100_diff1": 0.3097303489032699, + "nauc_map_at_100_max": 0.290298782604165, + "nauc_map_at_100_std": 0.06347144684839333, + "nauc_map_at_10_diff1": 0.3100470717224258, + "nauc_map_at_10_max": 0.28019036460754265, + "nauc_map_at_10_std": 0.04866509420826627, + "nauc_map_at_1_diff1": 0.37473679522803327, + "nauc_map_at_1_max": 0.28526104067645214, + "nauc_map_at_1_std": 0.004294440083795765, + "nauc_map_at_20_diff1": 0.3085372231530539, + "nauc_map_at_20_max": 0.2837468657125153, + "nauc_map_at_20_std": 0.054667743043272946, + "nauc_map_at_3_diff1": 0.32225252921254266, + "nauc_map_at_3_max": 0.27100397776424595, + "nauc_map_at_3_std": 0.03678088974870272, + "nauc_map_at_5_diff1": 0.3072216748439938, + "nauc_map_at_5_max": 0.2735756142039516, + "nauc_map_at_5_std": 0.0468454886538082, + "nauc_mrr_at_1000_diff1": 0.29037948385015405, + "nauc_mrr_at_1000_max": 0.28763423733914456, + "nauc_mrr_at_1000_std": 0.07956994665321418, + "nauc_mrr_at_100_diff1": 0.2900658551464532, + "nauc_mrr_at_100_max": 0.287893754933326, + "nauc_mrr_at_100_std": 0.0801121131116111, + "nauc_mrr_at_10_diff1": 0.29139613464995, + "nauc_mrr_at_10_max": 0.28581685095094267, + "nauc_mrr_at_10_std": 0.07614504456955483, + "nauc_mrr_at_1_diff1": 0.3363214806586087, + "nauc_mrr_at_1_max": 0.2880939584294624, + "nauc_mrr_at_1_std": 0.03870367319917128, + "nauc_mrr_at_20_diff1": 0.28897527412848617, + "nauc_mrr_at_20_max": 0.2844699870228015, + "nauc_mrr_at_20_std": 0.07611077545742706, + "nauc_mrr_at_3_diff1": 0.30195365305410854, + "nauc_mrr_at_3_max": 0.27875075763225104, + "nauc_mrr_at_3_std": 0.06392092075078198, + "nauc_mrr_at_5_diff1": 0.2902331736193772, + "nauc_mrr_at_5_max": 0.28526895681441516, + "nauc_mrr_at_5_std": 0.07505926748987359, + "nauc_ndcg_at_1000_diff1": 0.2949523126255678, + "nauc_ndcg_at_1000_max": 0.31136369251324425, + "nauc_ndcg_at_1000_std": 0.11458769315946413, + "nauc_ndcg_at_100_diff1": 0.2814154383832849, + "nauc_ndcg_at_100_max": 0.31317245100158053, + "nauc_ndcg_at_100_std": 0.12343874658257209, + "nauc_ndcg_at_10_diff1": 0.28551238661045275, + "nauc_ndcg_at_10_max": 0.28198394288716494, + "nauc_ndcg_at_10_std": 0.0699600068367413, + "nauc_ndcg_at_1_diff1": 0.3363214806586087, + "nauc_ndcg_at_1_max": 0.2880939584294624, + "nauc_ndcg_at_1_std": 0.03870367319917128, + "nauc_ndcg_at_20_diff1": 0.2806030705119219, + "nauc_ndcg_at_20_max": 0.2846993596667669, + "nauc_ndcg_at_20_std": 0.07854692180311396, + "nauc_ndcg_at_3_diff1": 0.30333015112698175, + "nauc_ndcg_at_3_max": 0.2725040631361372, + "nauc_ndcg_at_3_std": 0.05627330922520453, + "nauc_ndcg_at_5_diff1": 0.28450912289686897, + "nauc_ndcg_at_5_max": 0.277561662968176, + "nauc_ndcg_at_5_std": 0.07180600634098228, + "nauc_precision_at_1000_diff1": 0.054756151219749076, + "nauc_precision_at_1000_max": 0.12647020683088267, + "nauc_precision_at_1000_std": 0.13061720484832934, + "nauc_precision_at_100_diff1": 0.12347843899507294, + "nauc_precision_at_100_max": 0.2771737750802262, + "nauc_precision_at_100_std": 0.2673430752237529, + "nauc_precision_at_10_diff1": 0.1969177781807937, + "nauc_precision_at_10_max": 0.3033403080124182, + "nauc_precision_at_10_std": 0.15366237949875802, + "nauc_precision_at_1_diff1": 0.3363214806586087, + "nauc_precision_at_1_max": 0.2880939584294624, + "nauc_precision_at_1_std": 0.03870367319917128, + "nauc_precision_at_20_diff1": 0.1639420382179115, + "nauc_precision_at_20_max": 0.2853821894442003, + "nauc_precision_at_20_std": 0.1740002221290311, + "nauc_precision_at_3_diff1": 0.2501650633137198, + "nauc_precision_at_3_max": 0.2740140981400113, + "nauc_precision_at_3_std": 0.09528259668882319, + "nauc_precision_at_5_diff1": 0.20703008998648537, + "nauc_precision_at_5_max": 0.2902824629392723, + "nauc_precision_at_5_std": 0.14824369875578394, + "nauc_recall_at_1000_diff1": 0.23832697239355533, + "nauc_recall_at_1000_max": 0.4408435745523949, + "nauc_recall_at_1000_std": 0.4724836041537446, + "nauc_recall_at_100_diff1": 0.16300511580514368, + "nauc_recall_at_100_max": 0.35919938758349756, + "nauc_recall_at_100_std": 0.31711390146384005, + "nauc_recall_at_10_diff1": 0.2209945154154785, + "nauc_recall_at_10_max": 0.24889396522698598, + "nauc_recall_at_10_std": 0.08174508020746982, + "nauc_recall_at_1_diff1": 0.37473679522803327, + "nauc_recall_at_1_max": 0.28526104067645214, + "nauc_recall_at_1_std": 0.004294440083795765, + "nauc_recall_at_20_diff1": 0.20215222984918443, + "nauc_recall_at_20_max": 0.2429346815089969, + "nauc_recall_at_20_std": 0.09607537009954599, + "nauc_recall_at_3_diff1": 0.26891648077306185, + "nauc_recall_at_3_max": 0.24463956800351405, + "nauc_recall_at_3_std": 0.06593385364536652, + "nauc_recall_at_5_diff1": 0.22088516789906684, + "nauc_recall_at_5_max": 0.24856330338998195, + "nauc_recall_at_5_std": 0.08935178303242722, + "ndcg_at_1": 0.18151, + "ndcg_at_10": 0.27024, + "ndcg_at_100": 0.34035, + "ndcg_at_1000": 0.37122, + "ndcg_at_20": 0.29688, + "ndcg_at_3": 0.21837, + "ndcg_at_5": 0.23995, + "precision_at_1": 0.18151, + "precision_at_10": 0.05479, + "precision_at_100": 0.01083, + "precision_at_1000": 0.00154, + "precision_at_20": 0.03539, + "precision_at_3": 0.10693, + "precision_at_5": 0.08174, + "recall_at_1": 0.14198, + "recall_at_10": 0.38972, + "recall_at_100": 0.6897, + "recall_at_1000": 0.90291, + "recall_at_20": 0.48448, + "recall_at_3": 0.24421, + "recall_at_5": 0.30022 + } + ] + }, + "task_name": "CQADupstackProgrammersRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackStatsRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackStatsRetrieval.json new file mode 100644 index 000000000..59eb9529b --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackStatsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a", + "evaluation_time": 766.9537012577057, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.16322, + "map_at_1": 0.07804, + "map_at_10": 0.12989, + "map_at_100": 0.13857, + "map_at_1000": 0.14001, + "map_at_20": 0.13388, + "map_at_3": 0.11192, + "map_at_5": 0.12176, + "mrr_at_1": 0.09049079754601227, + "mrr_at_10": 0.14444809621189994, + "mrr_at_100": 0.15361749681377326, + "mrr_at_1000": 0.15483726108258947, + "mrr_at_20": 0.14891287287777494, + "mrr_at_3": 0.1273006134969325, + "mrr_at_5": 0.13665644171779143, + "nauc_map_at_1000_diff1": 0.22283889437078247, + "nauc_map_at_1000_max": 0.14944416577427388, + "nauc_map_at_1000_std": 0.07317249110014133, + "nauc_map_at_100_diff1": 0.2228273180365645, + "nauc_map_at_100_max": 0.14860297309097084, + "nauc_map_at_100_std": 0.07314010552135303, + "nauc_map_at_10_diff1": 0.22847655724529406, + "nauc_map_at_10_max": 0.15378238244826023, + "nauc_map_at_10_std": 0.07179195945065936, + "nauc_map_at_1_diff1": 0.3413458392270414, + "nauc_map_at_1_max": 0.17044590989203784, + "nauc_map_at_1_std": 0.003675928894683656, + "nauc_map_at_20_diff1": 0.22355665146014908, + "nauc_map_at_20_max": 0.14851743054913785, + "nauc_map_at_20_std": 0.07270308478997049, + "nauc_map_at_3_diff1": 0.24738705363741592, + "nauc_map_at_3_max": 0.1376122622256925, + "nauc_map_at_3_std": 0.04850569295830115, + "nauc_map_at_5_diff1": 0.23467560439858978, + "nauc_map_at_5_max": 0.14870499991558297, + "nauc_map_at_5_std": 0.0531865354060947, + "nauc_mrr_at_1000_diff1": 0.23134711618650636, + "nauc_mrr_at_1000_max": 0.15483448799540142, + "nauc_mrr_at_1000_std": 0.07296942679873371, + "nauc_mrr_at_100_diff1": 0.2311427541598592, + "nauc_mrr_at_100_max": 0.15449183219736304, + "nauc_mrr_at_100_std": 0.07319975335726775, + "nauc_mrr_at_10_diff1": 0.23488303603940713, + "nauc_mrr_at_10_max": 0.15817711277401347, + "nauc_mrr_at_10_std": 0.06933993376881063, + "nauc_mrr_at_1_diff1": 0.3331164870959659, + "nauc_mrr_at_1_max": 0.18141017780060012, + "nauc_mrr_at_1_std": 0.03542771244105074, + "nauc_mrr_at_20_diff1": 0.23174358926183958, + "nauc_mrr_at_20_max": 0.15311453081852944, + "nauc_mrr_at_20_std": 0.07194856706727489, + "nauc_mrr_at_3_diff1": 0.25424783972258036, + "nauc_mrr_at_3_max": 0.1554014171026033, + "nauc_mrr_at_3_std": 0.06762746993243182, + "nauc_mrr_at_5_diff1": 0.24246811872930796, + "nauc_mrr_at_5_max": 0.1562420277389364, + "nauc_mrr_at_5_std": 0.061118896243906, + "nauc_ndcg_at_1000_diff1": 0.1920841090878421, + "nauc_ndcg_at_1000_max": 0.1637452942677763, + "nauc_ndcg_at_1000_std": 0.10526588657800146, + "nauc_ndcg_at_100_diff1": 0.18922283216431407, + "nauc_ndcg_at_100_max": 0.14097138336254844, + "nauc_ndcg_at_100_std": 0.10389087631678794, + "nauc_ndcg_at_10_diff1": 0.19504289341336625, + "nauc_ndcg_at_10_max": 0.15117288827566233, + "nauc_ndcg_at_10_std": 0.09130368583693678, + "nauc_ndcg_at_1_diff1": 0.3331164870959659, + "nauc_ndcg_at_1_max": 0.18141017780060012, + "nauc_ndcg_at_1_std": 0.03542771244105074, + "nauc_ndcg_at_20_diff1": 0.18233271885408556, + "nauc_ndcg_at_20_max": 0.13419641851400083, + "nauc_ndcg_at_20_std": 0.09624024533607203, + "nauc_ndcg_at_3_diff1": 0.2229173448902654, + "nauc_ndcg_at_3_max": 0.1335741683411085, + "nauc_ndcg_at_3_std": 0.06620628886746967, + "nauc_ndcg_at_5_diff1": 0.20717344999706513, + "nauc_ndcg_at_5_max": 0.14469746539869008, + "nauc_ndcg_at_5_std": 0.06490356913357313, + "nauc_precision_at_1000_diff1": 0.03197341498400071, + "nauc_precision_at_1000_max": 0.2413255446383312, + "nauc_precision_at_1000_std": 0.14907711195306592, + "nauc_precision_at_100_diff1": 0.1056887700971008, + "nauc_precision_at_100_max": 0.18003165201051016, + "nauc_precision_at_100_std": 0.22326136388609719, + "nauc_precision_at_10_diff1": 0.13669635380022838, + "nauc_precision_at_10_max": 0.1979091772206062, + "nauc_precision_at_10_std": 0.19428145724937493, + "nauc_precision_at_1_diff1": 0.3331164870959659, + "nauc_precision_at_1_max": 0.18141017780060012, + "nauc_precision_at_1_std": 0.03542771244105074, + "nauc_precision_at_20_diff1": 0.1017191184815286, + "nauc_precision_at_20_max": 0.15434090781790694, + "nauc_precision_at_20_std": 0.20113197023271714, + "nauc_precision_at_3_diff1": 0.18663239563832687, + "nauc_precision_at_3_max": 0.15215643664801337, + "nauc_precision_at_3_std": 0.14073704225451486, + "nauc_precision_at_5_diff1": 0.15741117074117425, + "nauc_precision_at_5_max": 0.18083784453794294, + "nauc_precision_at_5_std": 0.14170154968425852, + "nauc_recall_at_1000_diff1": 0.12529462519944262, + "nauc_recall_at_1000_max": 0.2839083324666269, + "nauc_recall_at_1000_std": 0.22515746615902726, + "nauc_recall_at_100_diff1": 0.1331762885233605, + "nauc_recall_at_100_max": 0.11329638403083625, + "nauc_recall_at_100_std": 0.15910452829837185, + "nauc_recall_at_10_diff1": 0.11902493318826278, + "nauc_recall_at_10_max": 0.13620432998689277, + "nauc_recall_at_10_std": 0.12254249902080712, + "nauc_recall_at_1_diff1": 0.3413458392270414, + "nauc_recall_at_1_max": 0.17044590989203784, + "nauc_recall_at_1_std": 0.003675928894683656, + "nauc_recall_at_20_diff1": 0.09098713526218773, + "nauc_recall_at_20_max": 0.09041415083058396, + "nauc_recall_at_20_std": 0.13431500488643977, + "nauc_recall_at_3_diff1": 0.16135890777110928, + "nauc_recall_at_3_max": 0.0903348176243661, + "nauc_recall_at_3_std": 0.07396063204352615, + "nauc_recall_at_5_diff1": 0.14140180575248207, + "nauc_recall_at_5_max": 0.11845879266846512, + "nauc_recall_at_5_std": 0.070735189487104, + "ndcg_at_1": 0.09049, + "ndcg_at_10": 0.16322, + "ndcg_at_100": 0.21312, + "ndcg_at_1000": 0.25323, + "ndcg_at_20": 0.17763, + "ndcg_at_3": 0.12792, + "ndcg_at_5": 0.1445, + "precision_at_1": 0.09049, + "precision_at_10": 0.03083, + "precision_at_100": 0.00623, + "precision_at_1000": 0.00106, + "precision_at_20": 0.01902, + "precision_at_3": 0.06237, + "precision_at_5": 0.04785, + "recall_at_1": 0.07804, + "recall_at_10": 0.2528, + "recall_at_100": 0.49108, + "recall_at_1000": 0.79468, + "recall_at_20": 0.30605, + "recall_at_3": 0.15627, + "recall_at_5": 0.19529 + } + ] + }, + "task_name": "CQADupstackStatsRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackTexRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackTexRetrieval.json new file mode 100644 index 000000000..9704845c6 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackTexRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7", + "evaluation_time": 1393.736053943634, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.13762, + "map_at_1": 0.06376, + "map_at_10": 0.10558, + "map_at_100": 0.11532, + "map_at_1000": 0.11695, + "map_at_20": 0.11028, + "map_at_3": 0.08879, + "map_at_5": 0.09713, + "mrr_at_1": 0.08224363386097729, + "mrr_at_10": 0.13012431859644524, + "mrr_at_100": 0.13946814841802763, + "mrr_at_1000": 0.1406290442969105, + "mrr_at_20": 0.13497735030482463, + "mrr_at_3": 0.11155081440697408, + "mrr_at_5": 0.12115164028446872, + "nauc_map_at_1000_diff1": 0.2343507587077875, + "nauc_map_at_1000_max": 0.20438705647822789, + "nauc_map_at_1000_std": 0.12873082418993909, + "nauc_map_at_100_diff1": 0.23438050985565148, + "nauc_map_at_100_max": 0.20399313617924036, + "nauc_map_at_100_std": 0.12722487279959077, + "nauc_map_at_10_diff1": 0.24205691744325317, + "nauc_map_at_10_max": 0.20399611787740102, + "nauc_map_at_10_std": 0.12408592197191175, + "nauc_map_at_1_diff1": 0.340719849472863, + "nauc_map_at_1_max": 0.2235927499649868, + "nauc_map_at_1_std": 0.09086052394259114, + "nauc_map_at_20_diff1": 0.23544255806805497, + "nauc_map_at_20_max": 0.20248891696767904, + "nauc_map_at_20_std": 0.1251101167730077, + "nauc_map_at_3_diff1": 0.2648225696975319, + "nauc_map_at_3_max": 0.21208222413456188, + "nauc_map_at_3_std": 0.10787698162827862, + "nauc_map_at_5_diff1": 0.2511166942901193, + "nauc_map_at_5_max": 0.2084541422714762, + "nauc_map_at_5_std": 0.11084044604237207, + "nauc_mrr_at_1000_diff1": 0.21650853612058674, + "nauc_mrr_at_1000_max": 0.21417971870794875, + "nauc_mrr_at_1000_std": 0.15501648201831497, + "nauc_mrr_at_100_diff1": 0.2161532662748058, + "nauc_mrr_at_100_max": 0.21407856995308325, + "nauc_mrr_at_100_std": 0.15479695245125175, + "nauc_mrr_at_10_diff1": 0.22115984471726796, + "nauc_mrr_at_10_max": 0.21404557414782233, + "nauc_mrr_at_10_std": 0.15479987126659434, + "nauc_mrr_at_1_diff1": 0.3040287056679761, + "nauc_mrr_at_1_max": 0.2393938440196226, + "nauc_mrr_at_1_std": 0.12997536237455928, + "nauc_mrr_at_20_diff1": 0.21601037898265016, + "nauc_mrr_at_20_max": 0.21376546238813332, + "nauc_mrr_at_20_std": 0.15519106032577043, + "nauc_mrr_at_3_diff1": 0.2439132782582134, + "nauc_mrr_at_3_max": 0.22279324585023907, + "nauc_mrr_at_3_std": 0.1436723796962747, + "nauc_mrr_at_5_diff1": 0.22697790166099371, + "nauc_mrr_at_5_max": 0.21910155676991927, + "nauc_mrr_at_5_std": 0.14481452222550334, + "nauc_ndcg_at_1000_diff1": 0.18646473335235644, + "nauc_ndcg_at_1000_max": 0.2061450878843323, + "nauc_ndcg_at_1000_std": 0.1702503878660379, + "nauc_ndcg_at_100_diff1": 0.18516435075748475, + "nauc_ndcg_at_100_max": 0.19916591730147978, + "nauc_ndcg_at_100_std": 0.15365433816151405, + "nauc_ndcg_at_10_diff1": 0.20433623492398315, + "nauc_ndcg_at_10_max": 0.19505014482656055, + "nauc_ndcg_at_10_std": 0.15007091547236504, + "nauc_ndcg_at_1_diff1": 0.3040287056679761, + "nauc_ndcg_at_1_max": 0.2393938440196226, + "nauc_ndcg_at_1_std": 0.12997536237455928, + "nauc_ndcg_at_20_diff1": 0.18786357992932726, + "nauc_ndcg_at_20_max": 0.19098773770805913, + "nauc_ndcg_at_20_std": 0.15077078565809457, + "nauc_ndcg_at_3_diff1": 0.2372149188171607, + "nauc_ndcg_at_3_max": 0.21388331985304201, + "nauc_ndcg_at_3_std": 0.12898024573912106, + "nauc_ndcg_at_5_diff1": 0.21680662583678578, + "nauc_ndcg_at_5_max": 0.20613614404831923, + "nauc_ndcg_at_5_std": 0.12774272028171832, + "nauc_precision_at_1000_diff1": 0.009792326314146173, + "nauc_precision_at_1000_max": 0.18959462451645234, + "nauc_precision_at_1000_std": 0.27112339675868474, + "nauc_precision_at_100_diff1": 0.05507315634114162, + "nauc_precision_at_100_max": 0.2116546753595108, + "nauc_precision_at_100_std": 0.23890261020569653, + "nauc_precision_at_10_diff1": 0.12416990446480752, + "nauc_precision_at_10_max": 0.20972089855286186, + "nauc_precision_at_10_std": 0.23051845398601353, + "nauc_precision_at_1_diff1": 0.3040287056679761, + "nauc_precision_at_1_max": 0.2393938440196226, + "nauc_precision_at_1_std": 0.12997536237455928, + "nauc_precision_at_20_diff1": 0.0879422472044008, + "nauc_precision_at_20_max": 0.20674903676907636, + "nauc_precision_at_20_std": 0.23488209998563608, + "nauc_precision_at_3_diff1": 0.18022936966270942, + "nauc_precision_at_3_max": 0.22950256317548767, + "nauc_precision_at_3_std": 0.17001251788226637, + "nauc_precision_at_5_diff1": 0.1482261165916923, + "nauc_precision_at_5_max": 0.22371225776931078, + "nauc_precision_at_5_std": 0.17806150673765603, + "nauc_recall_at_1000_diff1": 0.07494489915270187, + "nauc_recall_at_1000_max": 0.20569342570845, + "nauc_recall_at_1000_std": 0.2586609450786989, + "nauc_recall_at_100_diff1": 0.1014085619919932, + "nauc_recall_at_100_max": 0.16931051365465138, + "nauc_recall_at_100_std": 0.15888964104887746, + "nauc_recall_at_10_diff1": 0.1422530728797227, + "nauc_recall_at_10_max": 0.1552986134762012, + "nauc_recall_at_10_std": 0.16717530917184517, + "nauc_recall_at_1_diff1": 0.340719849472863, + "nauc_recall_at_1_max": 0.2235927499649868, + "nauc_recall_at_1_std": 0.09086052394259114, + "nauc_recall_at_20_diff1": 0.10539561215408413, + "nauc_recall_at_20_max": 0.1454263335022052, + "nauc_recall_at_20_std": 0.16455084074160733, + "nauc_recall_at_3_diff1": 0.1986544236924314, + "nauc_recall_at_3_max": 0.18637208048866047, + "nauc_recall_at_3_std": 0.11885469722659006, + "nauc_recall_at_5_diff1": 0.16637796934858315, + "nauc_recall_at_5_max": 0.17833140476248197, + "nauc_recall_at_5_std": 0.12219010395478855, + "ndcg_at_1": 0.08224, + "ndcg_at_10": 0.13762, + "ndcg_at_100": 0.19006, + "ndcg_at_1000": 0.23144, + "ndcg_at_20": 0.15414, + "ndcg_at_3": 0.10464, + "ndcg_at_5": 0.11798, + "precision_at_1": 0.08224, + "precision_at_10": 0.0299, + "precision_at_100": 0.00689, + "precision_at_1000": 0.00124, + "precision_at_20": 0.01958, + "precision_at_3": 0.05368, + "precision_at_5": 0.04246, + "recall_at_1": 0.06376, + "recall_at_10": 0.21159, + "recall_at_100": 0.45506, + "recall_at_1000": 0.75702, + "recall_at_20": 0.27349, + "recall_at_3": 0.11837, + "recall_at_5": 0.15232 + } + ] + }, + "task_name": "CQADupstackTexRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackUnixRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackUnixRetrieval.json new file mode 100644 index 000000000..cdf164339 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "evaluation_time": 763.9203634262085, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2752, + "map_at_1": 0.15588, + "map_at_10": 0.22571, + "map_at_100": 0.23983, + "map_at_1000": 0.24122, + "map_at_20": 0.23391, + "map_at_3": 0.19767, + "map_at_5": 0.2118, + "mrr_at_1": 0.1875, + "mrr_at_10": 0.261608623548922, + "mrr_at_100": 0.2731701049716114, + "mrr_at_1000": 0.27405975383784187, + "mrr_at_20": 0.2687336000217942, + "mrr_at_3": 0.23445273631840813, + "mrr_at_5": 0.248212064676617, + "nauc_map_at_1000_diff1": 0.30684226668507025, + "nauc_map_at_1000_max": 0.24951065677373782, + "nauc_map_at_1000_std": 0.05206602668589683, + "nauc_map_at_100_diff1": 0.30679165862866475, + "nauc_map_at_100_max": 0.24953036476623802, + "nauc_map_at_100_std": 0.05205883565949459, + "nauc_map_at_10_diff1": 0.3112221536065615, + "nauc_map_at_10_max": 0.245124066227212, + "nauc_map_at_10_std": 0.0435292624287149, + "nauc_map_at_1_diff1": 0.3918247834731389, + "nauc_map_at_1_max": 0.24413907066366433, + "nauc_map_at_1_std": 0.02070371372630658, + "nauc_map_at_20_diff1": 0.30818586802122505, + "nauc_map_at_20_max": 0.24773933165290993, + "nauc_map_at_20_std": 0.04935672359462836, + "nauc_map_at_3_diff1": 0.3199317691078875, + "nauc_map_at_3_max": 0.23376773341655185, + "nauc_map_at_3_std": 0.017026914448918885, + "nauc_map_at_5_diff1": 0.315080910045294, + "nauc_map_at_5_max": 0.24560067495036395, + "nauc_map_at_5_std": 0.03552337846934037, + "nauc_mrr_at_1000_diff1": 0.28985987949736963, + "nauc_mrr_at_1000_max": 0.2702833053659213, + "nauc_mrr_at_1000_std": 0.06805769019477874, + "nauc_mrr_at_100_diff1": 0.28975617833163697, + "nauc_mrr_at_100_max": 0.2703219548034752, + "nauc_mrr_at_100_std": 0.06831150418834171, + "nauc_mrr_at_10_diff1": 0.29030864214165075, + "nauc_mrr_at_10_max": 0.2677447636537233, + "nauc_mrr_at_10_std": 0.06407525890746923, + "nauc_mrr_at_1_diff1": 0.36437744739184635, + "nauc_mrr_at_1_max": 0.2770874373843583, + "nauc_mrr_at_1_std": 0.04839051843867864, + "nauc_mrr_at_20_diff1": 0.29016976637513753, + "nauc_mrr_at_20_max": 0.270048771688745, + "nauc_mrr_at_20_std": 0.06775422720382143, + "nauc_mrr_at_3_diff1": 0.3007749391187041, + "nauc_mrr_at_3_max": 0.26340420675504694, + "nauc_mrr_at_3_std": 0.041417583286372114, + "nauc_mrr_at_5_diff1": 0.29434648381968587, + "nauc_mrr_at_5_max": 0.2704904363072304, + "nauc_mrr_at_5_std": 0.05638876519130594, + "nauc_ndcg_at_1000_diff1": 0.27201512870428224, + "nauc_ndcg_at_1000_max": 0.26607429208642486, + "nauc_ndcg_at_1000_std": 0.09426530569631955, + "nauc_ndcg_at_100_diff1": 0.267561672878664, + "nauc_ndcg_at_100_max": 0.26827656746985523, + "nauc_ndcg_at_100_std": 0.10117411449672364, + "nauc_ndcg_at_10_diff1": 0.2808039219869409, + "nauc_ndcg_at_10_max": 0.2530612250538394, + "nauc_ndcg_at_10_std": 0.06854192117532365, + "nauc_ndcg_at_1_diff1": 0.36437744739184635, + "nauc_ndcg_at_1_max": 0.2770874373843583, + "nauc_ndcg_at_1_std": 0.04839051843867864, + "nauc_ndcg_at_20_diff1": 0.2727399749691096, + "nauc_ndcg_at_20_max": 0.26107141341582335, + "nauc_ndcg_at_20_std": 0.08649506543910311, + "nauc_ndcg_at_3_diff1": 0.29424613119884024, + "nauc_ndcg_at_3_max": 0.24173847986413668, + "nauc_ndcg_at_3_std": 0.023404905076142166, + "nauc_ndcg_at_5_diff1": 0.2877369502430592, + "nauc_ndcg_at_5_max": 0.2564790019257201, + "nauc_ndcg_at_5_std": 0.05155072577708316, + "nauc_precision_at_1000_diff1": -0.047351598070789286, + "nauc_precision_at_1000_max": 0.038148394654088924, + "nauc_precision_at_1000_std": 0.05833941375739573, + "nauc_precision_at_100_diff1": 0.02744259727671696, + "nauc_precision_at_100_max": 0.21287933846054696, + "nauc_precision_at_100_std": 0.20177490390686853, + "nauc_precision_at_10_diff1": 0.15735811418920292, + "nauc_precision_at_10_max": 0.2722995218775853, + "nauc_precision_at_10_std": 0.14258744772316767, + "nauc_precision_at_1_diff1": 0.36437744739184635, + "nauc_precision_at_1_max": 0.2770874373843583, + "nauc_precision_at_1_std": 0.04839051843867864, + "nauc_precision_at_20_diff1": 0.1168099186992869, + "nauc_precision_at_20_max": 0.2711541433113338, + "nauc_precision_at_20_std": 0.1826747110990696, + "nauc_precision_at_3_diff1": 0.2210511897461343, + "nauc_precision_at_3_max": 0.2617867919555923, + "nauc_precision_at_3_std": 0.04707255449070724, + "nauc_precision_at_5_diff1": 0.19791048476417364, + "nauc_precision_at_5_max": 0.2923432771663703, + "nauc_precision_at_5_std": 0.10540070304399105, + "nauc_recall_at_1000_diff1": 0.08421470865964084, + "nauc_recall_at_1000_max": 0.35840684736708267, + "nauc_recall_at_1000_std": 0.438492485408352, + "nauc_recall_at_100_diff1": 0.14057683903084894, + "nauc_recall_at_100_max": 0.2918102905270184, + "nauc_recall_at_100_std": 0.26137740725020236, + "nauc_recall_at_10_diff1": 0.20883794816820972, + "nauc_recall_at_10_max": 0.23209326744403463, + "nauc_recall_at_10_std": 0.10744448122178654, + "nauc_recall_at_1_diff1": 0.3918247834731389, + "nauc_recall_at_1_max": 0.24413907066366433, + "nauc_recall_at_1_std": 0.02070371372630658, + "nauc_recall_at_20_diff1": 0.17615849661206665, + "nauc_recall_at_20_max": 0.25238448658847185, + "nauc_recall_at_20_std": 0.1641969229374057, + "nauc_recall_at_3_diff1": 0.24620971173135203, + "nauc_recall_at_3_max": 0.21570609052956133, + "nauc_recall_at_3_std": 0.01134113476733361, + "nauc_recall_at_5_diff1": 0.23026324115975752, + "nauc_recall_at_5_max": 0.2501133044022821, + "nauc_recall_at_5_std": 0.07490029675978573, + "ndcg_at_1": 0.1875, + "ndcg_at_10": 0.2752, + "ndcg_at_100": 0.34076, + "ndcg_at_1000": 0.37259, + "ndcg_at_20": 0.30266, + "ndcg_at_3": 0.22186, + "ndcg_at_5": 0.2439, + "precision_at_1": 0.1875, + "precision_at_10": 0.05177, + "precision_at_100": 0.00953, + "precision_at_1000": 0.00136, + "precision_at_20": 0.03279, + "precision_at_3": 0.10354, + "precision_at_5": 0.07743, + "recall_at_1": 0.15588, + "recall_at_10": 0.39202, + "recall_at_100": 0.68057, + "recall_at_1000": 0.90329, + "recall_at_20": 0.49331, + "recall_at_3": 0.24543, + "recall_at_5": 0.30133 + } + ] + }, + "task_name": "CQADupstackUnixRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWebmastersRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWebmastersRetrieval.json new file mode 100644 index 000000000..a877addca --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571", + "evaluation_time": 168.30577492713928, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.27418, + "map_at_1": 0.15928, + "map_at_10": 0.22445, + "map_at_100": 0.23729, + "map_at_1000": 0.23945, + "map_at_20": 0.23097, + "map_at_3": 0.19742, + "map_at_5": 0.21187, + "mrr_at_1": 0.1976284584980237, + "mrr_at_10": 0.2641194554238032, + "mrr_at_100": 0.275421765824062, + "mrr_at_1000": 0.27621234648152365, + "mrr_at_20": 0.2704753756334478, + "mrr_at_3": 0.23781291172595512, + "mrr_at_5": 0.25243741765480887, + "nauc_map_at_1000_diff1": 0.34875954784511165, + "nauc_map_at_1000_max": 0.29402608286733056, + "nauc_map_at_1000_std": 0.1888892400748289, + "nauc_map_at_100_diff1": 0.3467273650132461, + "nauc_map_at_100_max": 0.2936883910278622, + "nauc_map_at_100_std": 0.1880179375015822, + "nauc_map_at_10_diff1": 0.349259190155331, + "nauc_map_at_10_max": 0.2890614200989419, + "nauc_map_at_10_std": 0.1847240468202337, + "nauc_map_at_1_diff1": 0.41320257922377046, + "nauc_map_at_1_max": 0.27045525641366125, + "nauc_map_at_1_std": 0.14820719523595283, + "nauc_map_at_20_diff1": 0.3438606553153665, + "nauc_map_at_20_max": 0.29339130166316724, + "nauc_map_at_20_std": 0.18777126514716355, + "nauc_map_at_3_diff1": 0.36433641134537104, + "nauc_map_at_3_max": 0.28429893730930716, + "nauc_map_at_3_std": 0.16573807982457756, + "nauc_map_at_5_diff1": 0.3549829863276235, + "nauc_map_at_5_max": 0.2849206642708423, + "nauc_map_at_5_std": 0.17426074185877613, + "nauc_mrr_at_1000_diff1": 0.33688625969785524, + "nauc_mrr_at_1000_max": 0.3225195024978712, + "nauc_mrr_at_1000_std": 0.19920510902611938, + "nauc_mrr_at_100_diff1": 0.33696039879889045, + "nauc_mrr_at_100_max": 0.3226963439157751, + "nauc_mrr_at_100_std": 0.19963818748298817, + "nauc_mrr_at_10_diff1": 0.3396876056002211, + "nauc_mrr_at_10_max": 0.3210384558271738, + "nauc_mrr_at_10_std": 0.19742125901752847, + "nauc_mrr_at_1_diff1": 0.37824388683479415, + "nauc_mrr_at_1_max": 0.3302998941581047, + "nauc_mrr_at_1_std": 0.16995528862845455, + "nauc_mrr_at_20_diff1": 0.3359638068643063, + "nauc_mrr_at_20_max": 0.3230918088421744, + "nauc_mrr_at_20_std": 0.19976012832986972, + "nauc_mrr_at_3_diff1": 0.34672219815694805, + "nauc_mrr_at_3_max": 0.3194335032060688, + "nauc_mrr_at_3_std": 0.17957820211447814, + "nauc_mrr_at_5_diff1": 0.3416743584432419, + "nauc_mrr_at_5_max": 0.32025706005037724, + "nauc_mrr_at_5_std": 0.19012154164577907, + "nauc_ndcg_at_1000_diff1": 0.3282699931862427, + "nauc_ndcg_at_1000_max": 0.30765793533727215, + "nauc_ndcg_at_1000_std": 0.2204794628482662, + "nauc_ndcg_at_100_diff1": 0.3247557777891348, + "nauc_ndcg_at_100_max": 0.3091474769282945, + "nauc_ndcg_at_100_std": 0.22335685545222717, + "nauc_ndcg_at_10_diff1": 0.32682011590868465, + "nauc_ndcg_at_10_max": 0.2958944485699082, + "nauc_ndcg_at_10_std": 0.2144628383538255, + "nauc_ndcg_at_1_diff1": 0.37824388683479415, + "nauc_ndcg_at_1_max": 0.3302998941581047, + "nauc_ndcg_at_1_std": 0.16995528862845455, + "nauc_ndcg_at_20_diff1": 0.3092309131002204, + "nauc_ndcg_at_20_max": 0.3058371838446743, + "nauc_ndcg_at_20_std": 0.22307175012464037, + "nauc_ndcg_at_3_diff1": 0.3476421528647677, + "nauc_ndcg_at_3_max": 0.30087603886151176, + "nauc_ndcg_at_3_std": 0.187195367899851, + "nauc_ndcg_at_5_diff1": 0.3373582113410747, + "nauc_ndcg_at_5_max": 0.2939645291750342, + "nauc_ndcg_at_5_std": 0.19623919882911542, + "nauc_precision_at_1000_diff1": 0.32332205101930367, + "nauc_precision_at_1000_max": 0.09623028966785611, + "nauc_precision_at_1000_std": 0.10741264545736154, + "nauc_precision_at_100_diff1": 0.23981567728238543, + "nauc_precision_at_100_max": 0.22227294958441435, + "nauc_precision_at_100_std": 0.20635069438588186, + "nauc_precision_at_10_diff1": 0.22347016279919912, + "nauc_precision_at_10_max": 0.3158605136141047, + "nauc_precision_at_10_std": 0.26869528206483545, + "nauc_precision_at_1_diff1": 0.37824388683479415, + "nauc_precision_at_1_max": 0.3302998941581047, + "nauc_precision_at_1_std": 0.16995528862845455, + "nauc_precision_at_20_diff1": 0.18419666290305087, + "nauc_precision_at_20_max": 0.32321771618176115, + "nauc_precision_at_20_std": 0.2947795655526015, + "nauc_precision_at_3_diff1": 0.27863265709133367, + "nauc_precision_at_3_max": 0.35285838551821436, + "nauc_precision_at_3_std": 0.22672769836029127, + "nauc_precision_at_5_diff1": 0.25640021400595603, + "nauc_precision_at_5_max": 0.3324005429566627, + "nauc_precision_at_5_std": 0.2312118734680977, + "nauc_recall_at_1000_diff1": 0.12233272868637059, + "nauc_recall_at_1000_max": 0.3134004050188055, + "nauc_recall_at_1000_std": 0.3894602574688917, + "nauc_recall_at_100_diff1": 0.2347720048321073, + "nauc_recall_at_100_max": 0.3071446489259641, + "nauc_recall_at_100_std": 0.31928015150518757, + "nauc_recall_at_10_diff1": 0.2473037700905113, + "nauc_recall_at_10_max": 0.2646649780251438, + "nauc_recall_at_10_std": 0.2565417986790167, + "nauc_recall_at_1_diff1": 0.41320257922377046, + "nauc_recall_at_1_max": 0.27045525641366125, + "nauc_recall_at_1_std": 0.14820719523595283, + "nauc_recall_at_20_diff1": 0.1778335919785901, + "nauc_recall_at_20_max": 0.2891031250817357, + "nauc_recall_at_20_std": 0.28823907515459096, + "nauc_recall_at_3_diff1": 0.3141226901086654, + "nauc_recall_at_3_max": 0.262261413105601, + "nauc_recall_at_3_std": 0.17606845122657563, + "nauc_recall_at_5_diff1": 0.27631512319754586, + "nauc_recall_at_5_max": 0.2545630001064417, + "nauc_recall_at_5_std": 0.20554194945871143, + "ndcg_at_1": 0.19763, + "ndcg_at_10": 0.27418, + "ndcg_at_100": 0.33795, + "ndcg_at_1000": 0.37206, + "ndcg_at_20": 0.29497, + "ndcg_at_3": 0.22594, + "ndcg_at_5": 0.24782, + "precision_at_1": 0.19763, + "precision_at_10": 0.05415, + "precision_at_100": 0.01154, + "precision_at_1000": 0.0022, + "precision_at_20": 0.0335, + "precision_at_3": 0.10804, + "precision_at_5": 0.08221, + "recall_at_1": 0.15928, + "recall_at_10": 0.37972, + "recall_at_100": 0.68317, + "recall_at_1000": 0.89744, + "recall_at_20": 0.46399, + "recall_at_3": 0.23796, + "recall_at_5": 0.29528 + } + ] + }, + "task_name": "CQADupstackWebmastersRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWordpressRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWordpressRetrieval.json new file mode 100644 index 000000000..f5359e776 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/CQADupstackWordpressRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4", + "evaluation_time": 641.3694317340851, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2126, + "map_at_1": 0.10974, + "map_at_10": 0.17136, + "map_at_100": 0.18177, + "map_at_1000": 0.18287, + "map_at_20": 0.17636, + "map_at_3": 0.14934, + "map_at_5": 0.16073, + "mrr_at_1": 0.12199630314232902, + "mrr_at_10": 0.1870514626059913, + "mrr_at_100": 0.19748983258794034, + "mrr_at_1000": 0.19837442070088113, + "mrr_at_20": 0.19248603491362865, + "mrr_at_3": 0.16543438077634012, + "mrr_at_5": 0.17670979667282816, + "nauc_map_at_1000_diff1": 0.12678931653901057, + "nauc_map_at_1000_max": 0.16188177166378134, + "nauc_map_at_1000_std": 0.12840915498598127, + "nauc_map_at_100_diff1": 0.12670776495345507, + "nauc_map_at_100_max": 0.1617798043496996, + "nauc_map_at_100_std": 0.12822072072092977, + "nauc_map_at_10_diff1": 0.12288240379882705, + "nauc_map_at_10_max": 0.1615504212741907, + "nauc_map_at_10_std": 0.12164525297458954, + "nauc_map_at_1_diff1": 0.1907356377322629, + "nauc_map_at_1_max": 0.17999572753995083, + "nauc_map_at_1_std": 0.10302044048546005, + "nauc_map_at_20_diff1": 0.12310741859813562, + "nauc_map_at_20_max": 0.15874183751438678, + "nauc_map_at_20_std": 0.12329892661885576, + "nauc_map_at_3_diff1": 0.12923055965922023, + "nauc_map_at_3_max": 0.16030670921089712, + "nauc_map_at_3_std": 0.10174236364706073, + "nauc_map_at_5_diff1": 0.12334399032409554, + "nauc_map_at_5_max": 0.15847592809708466, + "nauc_map_at_5_std": 0.10642632496675083, + "nauc_mrr_at_1000_diff1": 0.14821689314895858, + "nauc_mrr_at_1000_max": 0.1801007338288078, + "nauc_mrr_at_1000_std": 0.12596964429973684, + "nauc_mrr_at_100_diff1": 0.14813075628659916, + "nauc_mrr_at_100_max": 0.17987900378330599, + "nauc_mrr_at_100_std": 0.12585770827205087, + "nauc_mrr_at_10_diff1": 0.14407724617483184, + "nauc_mrr_at_10_max": 0.179987939329453, + "nauc_mrr_at_10_std": 0.12295346246957185, + "nauc_mrr_at_1_diff1": 0.21868324878950887, + "nauc_mrr_at_1_max": 0.20894193908164335, + "nauc_mrr_at_1_std": 0.09139550923680906, + "nauc_mrr_at_20_diff1": 0.14570968072256132, + "nauc_mrr_at_20_max": 0.17744023038753384, + "nauc_mrr_at_20_std": 0.12120612537829208, + "nauc_mrr_at_3_diff1": 0.1573722761425632, + "nauc_mrr_at_3_max": 0.18197994422315622, + "nauc_mrr_at_3_std": 0.10196475686377902, + "nauc_mrr_at_5_diff1": 0.14745400930877595, + "nauc_mrr_at_5_max": 0.17907279586527572, + "nauc_mrr_at_5_std": 0.11138202351806359, + "nauc_ndcg_at_1000_diff1": 0.1213214616065359, + "nauc_ndcg_at_1000_max": 0.17121764777921208, + "nauc_ndcg_at_1000_std": 0.17893050819270823, + "nauc_ndcg_at_100_diff1": 0.11950146646556552, + "nauc_ndcg_at_100_max": 0.16509372959176655, + "nauc_ndcg_at_100_std": 0.17882671216027204, + "nauc_ndcg_at_10_diff1": 0.10280723820236314, + "nauc_ndcg_at_10_max": 0.1619992373296763, + "nauc_ndcg_at_10_std": 0.14772336098381986, + "nauc_ndcg_at_1_diff1": 0.21868324878950887, + "nauc_ndcg_at_1_max": 0.20894193908164335, + "nauc_ndcg_at_1_std": 0.09139550923680906, + "nauc_ndcg_at_20_diff1": 0.104067889502451, + "nauc_ndcg_at_20_max": 0.15234819276730865, + "nauc_ndcg_at_20_std": 0.14906728449046724, + "nauc_ndcg_at_3_diff1": 0.11811602705354898, + "nauc_ndcg_at_3_max": 0.16046989278985377, + "nauc_ndcg_at_3_std": 0.1067288489841697, + "nauc_ndcg_at_5_diff1": 0.10513005314547622, + "nauc_ndcg_at_5_max": 0.15506542272361076, + "nauc_ndcg_at_5_std": 0.1151643304336521, + "nauc_precision_at_1000_diff1": 0.06230871653536136, + "nauc_precision_at_1000_max": 0.028786312954363805, + "nauc_precision_at_1000_std": 0.08072311757002275, + "nauc_precision_at_100_diff1": 0.1269414692799929, + "nauc_precision_at_100_max": 0.15001693281013653, + "nauc_precision_at_100_std": 0.26434565117841186, + "nauc_precision_at_10_diff1": 0.07876949835492249, + "nauc_precision_at_10_max": 0.1651010130232648, + "nauc_precision_at_10_std": 0.20951244649430134, + "nauc_precision_at_1_diff1": 0.21868324878950887, + "nauc_precision_at_1_max": 0.20894193908164335, + "nauc_precision_at_1_std": 0.09139550923680906, + "nauc_precision_at_20_diff1": 0.08366060991850838, + "nauc_precision_at_20_max": 0.1254677453818851, + "nauc_precision_at_20_std": 0.20107820078217853, + "nauc_precision_at_3_diff1": 0.08338749776595783, + "nauc_precision_at_3_max": 0.1463768301600882, + "nauc_precision_at_3_std": 0.1133362167066395, + "nauc_precision_at_5_diff1": 0.06704572848424477, + "nauc_precision_at_5_max": 0.14248151818628482, + "nauc_precision_at_5_std": 0.13691110824662892, + "nauc_recall_at_1000_diff1": 0.0889266377200224, + "nauc_recall_at_1000_max": 0.2361464707731056, + "nauc_recall_at_1000_std": 0.48607983068257266, + "nauc_recall_at_100_diff1": 0.09377072086689382, + "nauc_recall_at_100_max": 0.15775682252417747, + "nauc_recall_at_100_std": 0.3386046204385918, + "nauc_recall_at_10_diff1": 0.040983133646851806, + "nauc_recall_at_10_max": 0.14716720941128567, + "nauc_recall_at_10_std": 0.20282626139543355, + "nauc_recall_at_1_diff1": 0.1907356377322629, + "nauc_recall_at_1_max": 0.17999572753995083, + "nauc_recall_at_1_std": 0.10302044048546005, + "nauc_recall_at_20_diff1": 0.04393976466517198, + "nauc_recall_at_20_max": 0.1195178800677615, + "nauc_recall_at_20_std": 0.20557023701011848, + "nauc_recall_at_3_diff1": 0.06734442526385427, + "nauc_recall_at_3_max": 0.14450825242271362, + "nauc_recall_at_3_std": 0.11589452715632739, + "nauc_recall_at_5_diff1": 0.04762756159503797, + "nauc_recall_at_5_max": 0.13251540766745998, + "nauc_recall_at_5_std": 0.12893569387731785, + "ndcg_at_1": 0.122, + "ndcg_at_10": 0.2126, + "ndcg_at_100": 0.26914, + "ndcg_at_1000": 0.30044, + "ndcg_at_20": 0.23145, + "ndcg_at_3": 0.16758, + "ndcg_at_5": 0.18715, + "precision_at_1": 0.122, + "precision_at_10": 0.03789, + "precision_at_100": 0.00719, + "precision_at_1000": 0.00106, + "precision_at_20": 0.02338, + "precision_at_3": 0.0764, + "precision_at_5": 0.0573, + "recall_at_1": 0.10974, + "recall_at_10": 0.32634, + "recall_at_100": 0.5921, + "recall_at_1000": 0.83458, + "recall_at_20": 0.39929, + "recall_at_3": 0.20283, + "recall_at_5": 0.24997 + } + ] + }, + "task_name": "CQADupstackWordpressRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/ClimateFEVER.json b/ai-sage__Giga-Embeddings-instruct/external/ClimateFEVER.json new file mode 100644 index 000000000..c28a4adfc --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "evaluation_time": 38328.236104249954, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1497, + "map_at_1": 0.04928, + "map_at_10": 0.0967, + "map_at_100": 0.11131, + "map_at_1000": 0.11361, + "map_at_20": 0.10248, + "map_at_3": 0.07931, + "map_at_5": 0.0878, + "mrr_at_1": 0.11856677524429968, + "mrr_at_10": 0.20355307378108659, + "mrr_at_100": 0.21828454101212888, + "mrr_at_1000": 0.21890691227973136, + "mrr_at_20": 0.21148065855052442, + "mrr_at_3": 0.17730727470141158, + "mrr_at_5": 0.19102062975027131, + "nauc_map_at_1000_diff1": 0.07343974864659536, + "nauc_map_at_1000_max": 0.2712797001490679, + "nauc_map_at_1000_std": 0.18688546971841116, + "nauc_map_at_100_diff1": 0.07257359641312383, + "nauc_map_at_100_max": 0.2693386501878897, + "nauc_map_at_100_std": 0.18572058476864395, + "nauc_map_at_10_diff1": 0.07552451155928329, + "nauc_map_at_10_max": 0.255823403439562, + "nauc_map_at_10_std": 0.16127814097430976, + "nauc_map_at_1_diff1": 0.12999418845432006, + "nauc_map_at_1_max": 0.31020337243647095, + "nauc_map_at_1_std": 0.19991896333019515, + "nauc_map_at_20_diff1": 0.07085975699404191, + "nauc_map_at_20_max": 0.2575407679129859, + "nauc_map_at_20_std": 0.16725260525712263, + "nauc_map_at_3_diff1": 0.0870936707305993, + "nauc_map_at_3_max": 0.26708517715732244, + "nauc_map_at_3_std": 0.1576063179131778, + "nauc_map_at_5_diff1": 0.0826315708475148, + "nauc_map_at_5_max": 0.2535727921546233, + "nauc_map_at_5_std": 0.150922582158579, + "nauc_mrr_at_1000_diff1": 0.09605823313934675, + "nauc_mrr_at_1000_max": 0.29210900493669495, + "nauc_mrr_at_1000_std": 0.1883080719056213, + "nauc_mrr_at_100_diff1": 0.0955666513233699, + "nauc_mrr_at_100_max": 0.2922087280589394, + "nauc_mrr_at_100_std": 0.18887436425893336, + "nauc_mrr_at_10_diff1": 0.09536442166623398, + "nauc_mrr_at_10_max": 0.2848930190421831, + "nauc_mrr_at_10_std": 0.17945183230798517, + "nauc_mrr_at_1_diff1": 0.1528049719426095, + "nauc_mrr_at_1_max": 0.34270497231895175, + "nauc_mrr_at_1_std": 0.21901296884315474, + "nauc_mrr_at_20_diff1": 0.09416526110965544, + "nauc_mrr_at_20_max": 0.2875113126415786, + "nauc_mrr_at_20_std": 0.18345484239730167, + "nauc_mrr_at_3_diff1": 0.10440304488887639, + "nauc_mrr_at_3_max": 0.294191822419378, + "nauc_mrr_at_3_std": 0.1779516933724722, + "nauc_mrr_at_5_diff1": 0.09933840128353966, + "nauc_mrr_at_5_max": 0.2838615077411181, + "nauc_mrr_at_5_std": 0.17568611963689923, + "nauc_ndcg_at_1000_diff1": 0.07472331359379457, + "nauc_ndcg_at_1000_max": 0.31639339374755143, + "nauc_ndcg_at_1000_std": 0.25105356776676657, + "nauc_ndcg_at_100_diff1": 0.06308931462956051, + "nauc_ndcg_at_100_max": 0.3065501104526197, + "nauc_ndcg_at_100_std": 0.2533133541833919, + "nauc_ndcg_at_10_diff1": 0.06318892234859429, + "nauc_ndcg_at_10_max": 0.2470336979572261, + "nauc_ndcg_at_10_std": 0.16190182720994473, + "nauc_ndcg_at_1_diff1": 0.1528049719426095, + "nauc_ndcg_at_1_max": 0.34270497231895175, + "nauc_ndcg_at_1_std": 0.21901296884315474, + "nauc_ndcg_at_20_diff1": 0.053618591994296465, + "nauc_ndcg_at_20_max": 0.25400570204198897, + "nauc_ndcg_at_20_std": 0.1788784418329067, + "nauc_ndcg_at_3_diff1": 0.08110953561431905, + "nauc_ndcg_at_3_max": 0.2663953855972143, + "nauc_ndcg_at_3_std": 0.15385397581839416, + "nauc_ndcg_at_5_diff1": 0.07709924887062054, + "nauc_ndcg_at_5_max": 0.24490998203146383, + "nauc_ndcg_at_5_std": 0.146921440386966, + "nauc_precision_at_1000_diff1": 0.036765328094092595, + "nauc_precision_at_1000_max": 0.22330333946150965, + "nauc_precision_at_1000_std": 0.22933547000215604, + "nauc_precision_at_100_diff1": 0.022666175638162384, + "nauc_precision_at_100_max": 0.30354739309785445, + "nauc_precision_at_100_std": 0.335620877250776, + "nauc_precision_at_10_diff1": 0.018399284347123332, + "nauc_precision_at_10_max": 0.21554621092765072, + "nauc_precision_at_10_std": 0.1612148112398472, + "nauc_precision_at_1_diff1": 0.1528049719426095, + "nauc_precision_at_1_max": 0.34270497231895175, + "nauc_precision_at_1_std": 0.21901296884315474, + "nauc_precision_at_20_diff1": 0.00019989740402583464, + "nauc_precision_at_20_max": 0.22184928806584195, + "nauc_precision_at_20_std": 0.19352421167092726, + "nauc_precision_at_3_diff1": 0.04993655109383327, + "nauc_precision_at_3_max": 0.25471904019950103, + "nauc_precision_at_3_std": 0.14428827762549512, + "nauc_precision_at_5_diff1": 0.04356666010145262, + "nauc_precision_at_5_max": 0.2193208543594649, + "nauc_precision_at_5_std": 0.1316874222343096, + "nauc_recall_at_1000_diff1": 0.07275822779465287, + "nauc_recall_at_1000_max": 0.3471656818418004, + "nauc_recall_at_1000_std": 0.34358417311322675, + "nauc_recall_at_100_diff1": 0.026813944416625025, + "nauc_recall_at_100_max": 0.30063206974414824, + "nauc_recall_at_100_std": 0.32997643399627635, + "nauc_recall_at_10_diff1": 0.02049411072425148, + "nauc_recall_at_10_max": 0.17384327223968482, + "nauc_recall_at_10_std": 0.12313062153156563, + "nauc_recall_at_1_diff1": 0.12999418845432006, + "nauc_recall_at_1_max": 0.31020337243647095, + "nauc_recall_at_1_std": 0.19991896333019515, + "nauc_recall_at_20_diff1": 0.000769739475464366, + "nauc_recall_at_20_max": 0.1840025074677139, + "nauc_recall_at_20_std": 0.1571519209254166, + "nauc_recall_at_3_diff1": 0.0589259927853003, + "nauc_recall_at_3_max": 0.2139147480034604, + "nauc_recall_at_3_std": 0.11573847953419673, + "nauc_recall_at_5_diff1": 0.049252507150816506, + "nauc_recall_at_5_max": 0.17681821165181708, + "nauc_recall_at_5_std": 0.10016468241533516, + "ndcg_at_1": 0.11857, + "ndcg_at_10": 0.1497, + "ndcg_at_100": 0.22719, + "ndcg_at_1000": 0.27034, + "ndcg_at_20": 0.17092, + "ndcg_at_3": 0.11584, + "ndcg_at_5": 0.12715, + "precision_at_1": 0.11857, + "precision_at_10": 0.04834, + "precision_at_100": 0.01336, + "precision_at_1000": 0.00212, + "precision_at_20": 0.03309, + "precision_at_3": 0.08903, + "precision_at_5": 0.0684, + "recall_at_1": 0.04928, + "recall_at_10": 0.19852, + "recall_at_100": 0.48075, + "recall_at_1000": 0.72608, + "recall_at_20": 0.25937, + "recall_at_3": 0.11309, + "recall_at_5": 0.14567 + } + ] + }, + "task_name": "ClimateFEVER" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/DBPedia.json b/ai-sage__Giga-Embeddings-instruct/external/DBPedia.json new file mode 100644 index 000000000..8a5e6baaf --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/DBPedia.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "evaluation_time": 20526.803413629532, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.30137, + "map_at_1": 0.06, + "map_at_10": 0.13453, + "map_at_100": 0.18389, + "map_at_1000": 0.19706, + "map_at_20": 0.15348, + "map_at_3": 0.0957, + "map_at_5": 0.11269, + "mrr_at_1": 0.485, + "mrr_at_10": 0.5912063492063492, + "mrr_at_100": 0.597067363969307, + "mrr_at_1000": 0.5973195255745748, + "mrr_at_20": 0.5949264190079787, + "mrr_at_3": 0.5679166666666667, + "mrr_at_5": 0.5810416666666667, + "nauc_map_at_1000_diff1": 0.2111048962823763, + "nauc_map_at_1000_max": 0.03824918831917913, + "nauc_map_at_1000_std": 0.18091642998292182, + "nauc_map_at_100_diff1": 0.21400896998453564, + "nauc_map_at_100_max": 0.030532938872682526, + "nauc_map_at_100_std": 0.15748553761470577, + "nauc_map_at_10_diff1": 0.21466273522809004, + "nauc_map_at_10_max": 0.010436684944087983, + "nauc_map_at_10_std": 0.031239808683438078, + "nauc_map_at_1_diff1": 0.3629495694371118, + "nauc_map_at_1_max": 0.05187621118221234, + "nauc_map_at_1_std": -0.02793466227392795, + "nauc_map_at_20_diff1": 0.22487809942004278, + "nauc_map_at_20_max": 0.01810008116855991, + "nauc_map_at_20_std": 0.06582415021321396, + "nauc_map_at_3_diff1": 0.2365507925300577, + "nauc_map_at_3_max": 0.015046652580013493, + "nauc_map_at_3_std": -0.007516301806524871, + "nauc_map_at_5_diff1": 0.21461287275511778, + "nauc_map_at_5_max": 0.013822213009694051, + "nauc_map_at_5_std": 0.00589954377858362, + "nauc_mrr_at_1000_diff1": 0.39959766604013414, + "nauc_mrr_at_1000_max": 0.14926325657707468, + "nauc_mrr_at_1000_std": 0.16345761016463395, + "nauc_mrr_at_100_diff1": 0.39959045716548836, + "nauc_mrr_at_100_max": 0.14934953971746828, + "nauc_mrr_at_100_std": 0.16376010454801465, + "nauc_mrr_at_10_diff1": 0.4006090715779124, + "nauc_mrr_at_10_max": 0.1488182049847337, + "nauc_mrr_at_10_std": 0.16592180496515538, + "nauc_mrr_at_1_diff1": 0.40879388159507973, + "nauc_mrr_at_1_max": 0.11270536619634297, + "nauc_mrr_at_1_std": 0.130413747854383, + "nauc_mrr_at_20_diff1": 0.40016477777174064, + "nauc_mrr_at_20_max": 0.14937806497015416, + "nauc_mrr_at_20_std": 0.1647207131196639, + "nauc_mrr_at_3_diff1": 0.39392818828294063, + "nauc_mrr_at_3_max": 0.14321953204624843, + "nauc_mrr_at_3_std": 0.13798281679964586, + "nauc_mrr_at_5_diff1": 0.3976897397855259, + "nauc_mrr_at_5_max": 0.14404530882907712, + "nauc_mrr_at_5_std": 0.1592346805944451, + "nauc_ndcg_at_1000_diff1": 0.22278226539846918, + "nauc_ndcg_at_1000_max": 0.09924226549383965, + "nauc_ndcg_at_1000_std": 0.2825033871964479, + "nauc_ndcg_at_100_diff1": 0.23500781065830162, + "nauc_ndcg_at_100_max": 0.06754347484202461, + "nauc_ndcg_at_100_std": 0.21003192623211656, + "nauc_ndcg_at_10_diff1": 0.26804949345594103, + "nauc_ndcg_at_10_max": 0.05287967810437971, + "nauc_ndcg_at_10_std": 0.15013863454844004, + "nauc_ndcg_at_1_diff1": 0.36514860068814176, + "nauc_ndcg_at_1_max": 0.10405773346698802, + "nauc_ndcg_at_1_std": 0.11799158787353368, + "nauc_ndcg_at_20_diff1": 0.27538405701690366, + "nauc_ndcg_at_20_max": 0.05803911184640402, + "nauc_ndcg_at_20_std": 0.14736490152112391, + "nauc_ndcg_at_3_diff1": 0.27617656142008573, + "nauc_ndcg_at_3_max": 0.08495506842062914, + "nauc_ndcg_at_3_std": 0.13000529342799003, + "nauc_ndcg_at_5_diff1": 0.2611786480923478, + "nauc_ndcg_at_5_max": 0.06810483365982102, + "nauc_ndcg_at_5_std": 0.13686050622685886, + "nauc_precision_at_1000_diff1": -0.018263159519304677, + "nauc_precision_at_1000_max": 0.09105654485044203, + "nauc_precision_at_1000_std": 0.19896196956194387, + "nauc_precision_at_100_diff1": 0.054217908480834374, + "nauc_precision_at_100_max": 0.05843754140943668, + "nauc_precision_at_100_std": 0.32872530636176, + "nauc_precision_at_10_diff1": 0.14266901547255353, + "nauc_precision_at_10_max": 0.05672058243815894, + "nauc_precision_at_10_std": 0.24731755691337357, + "nauc_precision_at_1_diff1": 0.40879388159507973, + "nauc_precision_at_1_max": 0.11270536619634297, + "nauc_precision_at_1_std": 0.130413747854383, + "nauc_precision_at_20_diff1": 0.14561505270649377, + "nauc_precision_at_20_max": 0.061877072435924364, + "nauc_precision_at_20_std": 0.28164304475344604, + "nauc_precision_at_3_diff1": 0.20903550544054858, + "nauc_precision_at_3_max": 0.08234045919189285, + "nauc_precision_at_3_std": 0.15882744817436434, + "nauc_precision_at_5_diff1": 0.1642363926772338, + "nauc_precision_at_5_max": 0.0669045337265027, + "nauc_precision_at_5_std": 0.1966413564319946, + "nauc_recall_at_1000_diff1": 0.0550000056743861, + "nauc_recall_at_1000_max": 0.06615548983596191, + "nauc_recall_at_1000_std": 0.3348096211301789, + "nauc_recall_at_100_diff1": 0.09888973775211436, + "nauc_recall_at_100_max": 0.027176483244916685, + "nauc_recall_at_100_std": 0.20844187088199875, + "nauc_recall_at_10_diff1": 0.16134031012199307, + "nauc_recall_at_10_max": -0.004718490485814091, + "nauc_recall_at_10_std": 0.021253602607327875, + "nauc_recall_at_1_diff1": 0.3629495694371118, + "nauc_recall_at_1_max": 0.05187621118221234, + "nauc_recall_at_1_std": -0.02793466227392795, + "nauc_recall_at_20_diff1": 0.1948409505091461, + "nauc_recall_at_20_max": 0.03295168793698709, + "nauc_recall_at_20_std": 0.0700681132960327, + "nauc_recall_at_3_diff1": 0.18267617168970432, + "nauc_recall_at_3_max": -0.0006449790862866218, + "nauc_recall_at_3_std": -0.030161587054328993, + "nauc_recall_at_5_diff1": 0.14913123369125839, + "nauc_recall_at_5_max": 0.0016850104026158686, + "nauc_recall_at_5_std": -0.006034095216457759, + "ndcg_at_1": 0.385, + "ndcg_at_10": 0.30137, + "ndcg_at_100": 0.34342, + "ndcg_at_1000": 0.42201, + "ndcg_at_20": 0.29822, + "ndcg_at_3": 0.32983, + "ndcg_at_5": 0.31182, + "precision_at_1": 0.485, + "precision_at_10": 0.241, + "precision_at_100": 0.07605, + "precision_at_1000": 0.01733, + "precision_at_20": 0.18137, + "precision_at_3": 0.36083, + "precision_at_5": 0.3075, + "recall_at_1": 0.06, + "recall_at_10": 0.19136, + "recall_at_100": 0.41338, + "recall_at_1000": 0.6725, + "recall_at_20": 0.24688, + "recall_at_3": 0.11132, + "recall_at_5": 0.14231 + } + ] + }, + "task_name": "DBPedia" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/EmotionClassification.json b/ai-sage__Giga-Embeddings-instruct/external/EmotionClassification.json new file mode 100644 index 000000000..e7b7d6302 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/EmotionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", + "evaluation_time": 105.17276787757874, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.7309500000000001, + "f1": 0.6863641879623752, + "f1_weighted": 0.7408687647970862, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7309500000000001, + "scores_per_experiment": [ + { + "accuracy": 0.738, + "f1": 0.6814649343649882, + "f1_weighted": 0.7487588972136352 + }, + { + "accuracy": 0.735, + "f1": 0.693240402106869, + "f1_weighted": 0.7428503273179617 + }, + { + "accuracy": 0.7245, + "f1": 0.6819373851529599, + "f1_weighted": 0.7341645641172396 + }, + { + "accuracy": 0.728, + "f1": 0.685338051381665, + "f1_weighted": 0.7396947208760294 + }, + { + "accuracy": 0.718, + "f1": 0.668466370917986, + "f1_weighted": 0.7313404883513269 + }, + { + "accuracy": 0.7365, + "f1": 0.6914082144932271, + "f1_weighted": 0.7449846810871401 + }, + { + "accuracy": 0.7385, + "f1": 0.6952225025103407, + "f1_weighted": 0.7452725084929154 + }, + { + "accuracy": 0.734, + "f1": 0.6913400349684481, + "f1_weighted": 0.7436025875829413 + }, + { + "accuracy": 0.738, + "f1": 0.6935812782453304, + "f1_weighted": 0.7482464048657479 + }, + { + "accuracy": 0.719, + "f1": 0.681642705481938, + "f1_weighted": 0.7297724680659251 + } + ] + } + ] + }, + "task_name": "EmotionClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/FEVER.json b/ai-sage__Giga-Embeddings-instruct/external/FEVER.json new file mode 100644 index 000000000..bab5bdc11 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/FEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "evaluation_time": 38600.496267557144, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56275, + "map_at_1": 0.35847, + "map_at_10": 0.49192, + "map_at_100": 0.49913, + "map_at_1000": 0.49945, + "map_at_20": 0.49652, + "map_at_3": 0.45988, + "map_at_5": 0.47955, + "mrr_at_1": 0.385988598859886, + "mrr_at_10": 0.5223110406278746, + "mrr_at_100": 0.5288492739273575, + "mrr_at_1000": 0.5290474710635739, + "mrr_at_20": 0.526636924268024, + "mrr_at_3": 0.48972397239724275, + "mrr_at_5": 0.5101635163516368, + "nauc_map_at_1000_diff1": 0.3664174970870812, + "nauc_map_at_1000_max": 0.15660507126063872, + "nauc_map_at_1000_std": -0.12525150655309775, + "nauc_map_at_100_diff1": 0.366435791136361, + "nauc_map_at_100_max": 0.15675834344014947, + "nauc_map_at_100_std": -0.12504087019254898, + "nauc_map_at_10_diff1": 0.36512625030119367, + "nauc_map_at_10_max": 0.15521396090488226, + "nauc_map_at_10_std": -0.12655309941517057, + "nauc_map_at_1_diff1": 0.38194017541516573, + "nauc_map_at_1_max": 0.13290047228169902, + "nauc_map_at_1_std": -0.1625017425222326, + "nauc_map_at_20_diff1": 0.3661973919620995, + "nauc_map_at_20_max": 0.1567987191185896, + "nauc_map_at_20_std": -0.12487736624487598, + "nauc_map_at_3_diff1": 0.3664249275169512, + "nauc_map_at_3_max": 0.14555494484309506, + "nauc_map_at_3_std": -0.1477813664231636, + "nauc_map_at_5_diff1": 0.3660147619857385, + "nauc_map_at_5_max": 0.15091571640821227, + "nauc_map_at_5_std": -0.13570700587891282, + "nauc_mrr_at_1000_diff1": 0.3799742643197941, + "nauc_mrr_at_1000_max": 0.1498013800807032, + "nauc_mrr_at_1000_std": -0.14841942787676668, + "nauc_mrr_at_100_diff1": 0.38000767696175036, + "nauc_mrr_at_100_max": 0.14996219532747035, + "nauc_mrr_at_100_std": -0.14814824444455868, + "nauc_mrr_at_10_diff1": 0.37804918077031896, + "nauc_mrr_at_10_max": 0.14882959946615573, + "nauc_mrr_at_10_std": -0.1487210289207, + "nauc_mrr_at_1_diff1": 0.3930857248856019, + "nauc_mrr_at_1_max": 0.13216181318724654, + "nauc_mrr_at_1_std": -0.17816031667670962, + "nauc_mrr_at_20_diff1": 0.3795832753928953, + "nauc_mrr_at_20_max": 0.1501939060882702, + "nauc_mrr_at_20_std": -0.14742321408684889, + "nauc_mrr_at_3_diff1": 0.3783776099785701, + "nauc_mrr_at_3_max": 0.13991066447397454, + "nauc_mrr_at_3_std": -0.16852370243294917, + "nauc_mrr_at_5_diff1": 0.37802413999407086, + "nauc_mrr_at_5_max": 0.14504998552455786, + "nauc_mrr_at_5_std": -0.15653347947582472, + "nauc_ndcg_at_1000_diff1": 0.36657565801237485, + "nauc_ndcg_at_1000_max": 0.16974447484849262, + "nauc_ndcg_at_1000_std": -0.09948608541675864, + "nauc_ndcg_at_100_diff1": 0.3671481365126713, + "nauc_ndcg_at_100_max": 0.17489332769921878, + "nauc_ndcg_at_100_std": -0.0911867227279014, + "nauc_ndcg_at_10_diff1": 0.36067999615343144, + "nauc_ndcg_at_10_max": 0.16861367700049498, + "nauc_ndcg_at_10_std": -0.0981556106125859, + "nauc_ndcg_at_1_diff1": 0.3930857248856019, + "nauc_ndcg_at_1_max": 0.13216181318724654, + "nauc_ndcg_at_1_std": -0.17816031667670962, + "nauc_ndcg_at_20_diff1": 0.3649965813253492, + "nauc_ndcg_at_20_max": 0.17539102339776294, + "nauc_ndcg_at_20_std": -0.09026565538786357, + "nauc_ndcg_at_3_diff1": 0.36418158396935885, + "nauc_ndcg_at_3_max": 0.14700854622435058, + "nauc_ndcg_at_3_std": -0.14674234384559073, + "nauc_ndcg_at_5_diff1": 0.3628943043198077, + "nauc_ndcg_at_5_max": 0.1574587938209928, + "nauc_ndcg_at_5_std": -0.1224120367991547, + "nauc_precision_at_1000_diff1": -0.011251650712947921, + "nauc_precision_at_1000_max": 0.06995559652372453, + "nauc_precision_at_1000_std": 0.12831131017541822, + "nauc_precision_at_100_diff1": 0.11000701192784644, + "nauc_precision_at_100_max": 0.20360887483968634, + "nauc_precision_at_100_std": 0.2216846897480723, + "nauc_precision_at_10_diff1": 0.2700078852057367, + "nauc_precision_at_10_max": 0.21829143356067338, + "nauc_precision_at_10_std": 0.05798370950077175, + "nauc_precision_at_1_diff1": 0.3930857248856019, + "nauc_precision_at_1_max": 0.13216181318724654, + "nauc_precision_at_1_std": -0.17816031667670962, + "nauc_precision_at_20_diff1": 0.24362454958286267, + "nauc_precision_at_20_max": 0.25723944313836017, + "nauc_precision_at_20_std": 0.14965719248393347, + "nauc_precision_at_3_diff1": 0.3379887736757885, + "nauc_precision_at_3_max": 0.1536365542882353, + "nauc_precision_at_3_std": -0.1379108291639478, + "nauc_precision_at_5_diff1": 0.319565627315705, + "nauc_precision_at_5_max": 0.18233368709013878, + "nauc_precision_at_5_std": -0.06431537681620625, + "nauc_recall_at_1000_diff1": 0.18340052781682273, + "nauc_recall_at_1000_max": 0.31963275437350025, + "nauc_recall_at_1000_std": 0.44816169836161657, + "nauc_recall_at_100_diff1": 0.2822528558216003, + "nauc_recall_at_100_max": 0.345813678838198, + "nauc_recall_at_100_std": 0.3467537364886928, + "nauc_recall_at_10_diff1": 0.297947859758469, + "nauc_recall_at_10_max": 0.2249573322191089, + "nauc_recall_at_10_std": 0.07132599572662986, + "nauc_recall_at_1_diff1": 0.38194017541516573, + "nauc_recall_at_1_max": 0.13290047228169902, + "nauc_recall_at_1_std": -0.1625017425222326, + "nauc_recall_at_20_diff1": 0.3026074789910862, + "nauc_recall_at_20_max": 0.27834226251757815, + "nauc_recall_at_20_std": 0.16548411990150652, + "nauc_recall_at_3_diff1": 0.33413742439395483, + "nauc_recall_at_3_max": 0.15211407121882475, + "nauc_recall_at_3_std": -0.11671768186203062, + "nauc_recall_at_5_diff1": 0.3221235453464061, + "nauc_recall_at_5_max": 0.1782957773085371, + "nauc_recall_at_5_std": -0.04588112602146949, + "ndcg_at_1": 0.38599, + "ndcg_at_10": 0.56275, + "ndcg_at_100": 0.59581, + "ndcg_at_1000": 0.60307, + "ndcg_at_20": 0.57865, + "ndcg_at_3": 0.49903, + "ndcg_at_5": 0.53378, + "precision_at_1": 0.38599, + "precision_at_10": 0.08237, + "precision_at_100": 0.01004, + "precision_at_1000": 0.00108, + "precision_at_20": 0.0447, + "precision_at_3": 0.20982, + "precision_at_5": 0.14476, + "recall_at_1": 0.35847, + "recall_at_10": 0.75244, + "recall_at_100": 0.89968, + "recall_at_1000": 0.95281, + "recall_at_20": 0.81272, + "recall_at_3": 0.58149, + "recall_at_5": 0.6653 + } + ] + }, + "task_name": "FEVER" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/FiQA2018.json b/ai-sage__Giga-Embeddings-instruct/external/FiQA2018.json new file mode 100644 index 000000000..c4460ad96 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "evaluation_time": 530.1675961017609, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7356, + "map_at_1": 0.36149, + "map_at_10": 0.64262, + "map_at_100": 0.66346, + "map_at_1000": 0.66381, + "map_at_20": 0.6591, + "map_at_3": 0.54579, + "map_at_5": 0.60043, + "mrr_at_1": 0.6820987654320988, + "mrr_at_10": 0.7806761953752689, + "mrr_at_100": 0.7822788481128512, + "mrr_at_1000": 0.782351762841002, + "mrr_at_20": 0.7819418362303349, + "mrr_at_3": 0.7638888888888885, + "mrr_at_5": 0.7753858024691351, + "nauc_map_at_1000_diff1": 0.3913951115792765, + "nauc_map_at_1000_max": 0.1578967564149681, + "nauc_map_at_1000_std": -0.14962767430758966, + "nauc_map_at_100_diff1": 0.3914667307112649, + "nauc_map_at_100_max": 0.157556118076197, + "nauc_map_at_100_std": -0.14947192943164422, + "nauc_map_at_10_diff1": 0.3971365566339051, + "nauc_map_at_10_max": 0.12359982700266224, + "nauc_map_at_10_std": -0.16689077406647765, + "nauc_map_at_1_diff1": 0.3801390755497353, + "nauc_map_at_1_max": -0.024285798001758695, + "nauc_map_at_1_std": -0.11839428013172744, + "nauc_map_at_20_diff1": 0.3927021403147416, + "nauc_map_at_20_max": 0.1522857604958874, + "nauc_map_at_20_std": -0.1531472460777032, + "nauc_map_at_3_diff1": 0.379774760696472, + "nauc_map_at_3_max": 0.03219061299660823, + "nauc_map_at_3_std": -0.1566245768446722, + "nauc_map_at_5_diff1": 0.394541015155147, + "nauc_map_at_5_max": 0.08406851080721177, + "nauc_map_at_5_std": -0.17318320201143725, + "nauc_mrr_at_1000_diff1": 0.5052179630360454, + "nauc_mrr_at_1000_max": 0.27958043634255975, + "nauc_mrr_at_1000_std": -0.10449298885386217, + "nauc_mrr_at_100_diff1": 0.5051660811731795, + "nauc_mrr_at_100_max": 0.27944402723175604, + "nauc_mrr_at_100_std": -0.1044533164401012, + "nauc_mrr_at_10_diff1": 0.5043596746653215, + "nauc_mrr_at_10_max": 0.27849753656390963, + "nauc_mrr_at_10_std": -0.10448303695173873, + "nauc_mrr_at_1_diff1": 0.5229547373761976, + "nauc_mrr_at_1_max": 0.2884554542629197, + "nauc_mrr_at_1_std": -0.09753666425872645, + "nauc_mrr_at_20_diff1": 0.5047556830247787, + "nauc_mrr_at_20_max": 0.2792615740500234, + "nauc_mrr_at_20_std": -0.10413757833872395, + "nauc_mrr_at_3_diff1": 0.5030231795753932, + "nauc_mrr_at_3_max": 0.2723788784979687, + "nauc_mrr_at_3_std": -0.1152516432450595, + "nauc_mrr_at_5_diff1": 0.4978580615169249, + "nauc_mrr_at_5_max": 0.2846933101763155, + "nauc_mrr_at_5_std": -0.10669870915511963, + "nauc_ndcg_at_1000_diff1": 0.4121630766413613, + "nauc_ndcg_at_1000_max": 0.21898896244340968, + "nauc_ndcg_at_1000_std": -0.11887919577706998, + "nauc_ndcg_at_100_diff1": 0.4131739592194364, + "nauc_ndcg_at_100_max": 0.21604565329954767, + "nauc_ndcg_at_100_std": -0.11487948873516628, + "nauc_ndcg_at_10_diff1": 0.41478136363299944, + "nauc_ndcg_at_10_max": 0.16261718498378902, + "nauc_ndcg_at_10_std": -0.14885574869807708, + "nauc_ndcg_at_1_diff1": 0.5229547373761976, + "nauc_ndcg_at_1_max": 0.2884554542629197, + "nauc_ndcg_at_1_std": -0.09753666425872645, + "nauc_ndcg_at_20_diff1": 0.41066932748173796, + "nauc_ndcg_at_20_max": 0.20514680273626568, + "nauc_ndcg_at_20_std": -0.12435494727134183, + "nauc_ndcg_at_3_diff1": 0.3683325361853425, + "nauc_ndcg_at_3_max": 0.1738656549225942, + "nauc_ndcg_at_3_std": -0.13019339397502755, + "nauc_ndcg_at_5_diff1": 0.38580162281647057, + "nauc_ndcg_at_5_max": 0.16337906272293548, + "nauc_ndcg_at_5_std": -0.15684185539933096, + "nauc_precision_at_1000_diff1": -0.15015876704383985, + "nauc_precision_at_1000_max": 0.315974559756291, + "nauc_precision_at_1000_std": 0.10381684525984322, + "nauc_precision_at_100_diff1": -0.13999569979209717, + "nauc_precision_at_100_max": 0.32073811750318315, + "nauc_precision_at_100_std": 0.10894062912630724, + "nauc_precision_at_10_diff1": -0.06838189677762939, + "nauc_precision_at_10_max": 0.308435231260984, + "nauc_precision_at_10_std": 0.03584893296243004, + "nauc_precision_at_1_diff1": 0.5229547373761976, + "nauc_precision_at_1_max": 0.2884554542629197, + "nauc_precision_at_1_std": -0.09753666425872645, + "nauc_precision_at_20_diff1": -0.11288106674270978, + "nauc_precision_at_20_max": 0.32833427715252383, + "nauc_precision_at_20_std": 0.08381049941157823, + "nauc_precision_at_3_diff1": 0.08995935650409244, + "nauc_precision_at_3_max": 0.2860491635762267, + "nauc_precision_at_3_std": -0.034130115967090424, + "nauc_precision_at_5_diff1": 0.0050963331859041954, + "nauc_precision_at_5_max": 0.3164022178781689, + "nauc_precision_at_5_std": -0.02229213295556884, + "nauc_recall_at_1000_diff1": 0.11836577229926257, + "nauc_recall_at_1000_max": 0.632069044893288, + "nauc_recall_at_1000_std": 0.39482784481120176, + "nauc_recall_at_100_diff1": 0.33513987961515096, + "nauc_recall_at_100_max": 0.22125177627882647, + "nauc_recall_at_100_std": 0.14591263016360373, + "nauc_recall_at_10_diff1": 0.3236111021259031, + "nauc_recall_at_10_max": 0.018443323783196738, + "nauc_recall_at_10_std": -0.15156002529122314, + "nauc_recall_at_1_diff1": 0.3801390755497353, + "nauc_recall_at_1_max": -0.024285798001758695, + "nauc_recall_at_1_std": -0.11839428013172744, + "nauc_recall_at_20_diff1": 0.2809780616502855, + "nauc_recall_at_20_max": 0.16429971434661592, + "nauc_recall_at_20_std": -0.024852013016206157, + "nauc_recall_at_3_diff1": 0.3125125120179455, + "nauc_recall_at_3_max": -0.03973563562137874, + "nauc_recall_at_3_std": -0.15354908114563473, + "nauc_recall_at_5_diff1": 0.3124816861720535, + "nauc_recall_at_5_max": 0.02908586508028948, + "nauc_recall_at_5_std": -0.16221156997636046, + "ndcg_at_1": 0.6821, + "ndcg_at_10": 0.7356, + "ndcg_at_100": 0.77522, + "ndcg_at_1000": 0.77953, + "ndcg_at_20": 0.76193, + "ndcg_at_3": 0.66853, + "ndcg_at_5": 0.69602, + "precision_at_1": 0.6821, + "precision_at_10": 0.21142, + "precision_at_100": 0.02546, + "precision_at_1000": 0.00262, + "precision_at_20": 0.11914, + "precision_at_3": 0.45525, + "precision_at_5": 0.3429, + "recall_at_1": 0.36149, + "recall_at_10": 0.84102, + "recall_at_100": 0.96749, + "recall_at_1000": 0.99341, + "recall_at_20": 0.91662, + "recall_at_3": 0.61603, + "recall_at_5": 0.72482 + } + ] + }, + "task_name": "FiQA2018" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClassification.json b/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClassification.json new file mode 100644 index 000000000..53c5984c2 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", "evaluation_time": 129.47752165794373, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.5859296875, "f1": 0.5587139132757291, "f1_weighted": 0.5586634332171304, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.5859296875, "scores_per_experiment": [{"accuracy": 0.55731640625, "f1": 0.5229694294118136, "f1_weighted": 0.5229222701859639}, {"accuracy": 0.591984375, "f1": 0.555924901425645, "f1_weighted": 0.5558211023944152}, {"accuracy": 0.586125, "f1": 0.5494325057284366, "f1_weighted": 0.5493564211467173}, {"accuracy": 0.5792890625, "f1": 0.5501648745757316, "f1_weighted": 0.5501707550356862}, {"accuracy": 0.59149609375, "f1": 0.5648361081333897, "f1_weighted": 0.5647721231475495}, {"accuracy": 0.58173046875, "f1": 0.5636079375892885, "f1_weighted": 0.5635801978534265}, {"accuracy": 0.61005078125, "f1": 0.5834760059231291, "f1_weighted": 0.5834127527440054}, {"accuracy": 0.5851484375, "f1": 0.5616457149561589, "f1_weighted": 0.5615781615536618}, {"accuracy": 0.5783125, "f1": 0.5636164019286092, "f1_weighted": 0.5635985547444929}, {"accuracy": 0.59784375, "f1": 0.571465253085089, "f1_weighted": 0.5714219933653855}]}]}, "task_name": "GeoreviewClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClusteringP2P.json b/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..885fb18b6 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 74.29855298995972, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6137882703448214, + "v_measure": 0.6137882703448214, + "v_measure_std": 0.00903844050224755, + "v_measures": { + "Level 0": [ + 0.6194106379207763, + 0.5987663324293994, + 0.6160008021171163, + 0.6234102542383854, + 0.6161666616636329, + 0.5950979445020341, + 0.6167983303408869, + 0.611154005094109, + 0.620712928705099, + 0.6203648064367759 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/HeadlineClassification.json b/ai-sage__Giga-Embeddings-instruct/external/HeadlineClassification.json new file mode 100644 index 000000000..5ea538944 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/HeadlineClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", "evaluation_time": 119.27487397193909, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.8601484375, "f1": 0.8517323995821744, "f1_weighted": 0.8517273609511149, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.8601484375, "scores_per_experiment": [{"accuracy": 0.8390546875, "f1": 0.8310858988529732, "f1_weighted": 0.831093128291411}, {"accuracy": 0.8663984375, "f1": 0.8575532533424209, "f1_weighted": 0.8575438533546816}, {"accuracy": 0.85614453125, "f1": 0.8484491700268699, "f1_weighted": 0.8484410226975732}, {"accuracy": 0.86200390625, "f1": 0.8533053929635098, "f1_weighted": 0.8532982792561661}, {"accuracy": 0.87274609375, "f1": 0.8649271365066235, "f1_weighted": 0.8649164965450048}, {"accuracy": 0.857609375, "f1": 0.8496922977084708, "f1_weighted": 0.8496674849408536}, {"accuracy": 0.86395703125, "f1": 0.8556173814096782, "f1_weighted": 0.8556107924249053}, {"accuracy": 0.86395703125, "f1": 0.8551599807673854, "f1_weighted": 0.8551692881368087}, {"accuracy": 0.8488203125, "f1": 0.8393928418867795, "f1_weighted": 0.839392535986486}, {"accuracy": 0.87079296875, "f1": 0.8621406423570331, "f1_weighted": 0.862140727877259}]}]}, "task_name": "HeadlineClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/HotpotQA.json b/ai-sage__Giga-Embeddings-instruct/external/HotpotQA.json new file mode 100644 index 000000000..e8fcf966c --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/HotpotQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "evaluation_time": 22361.053863048553, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56936, + "map_at_1": 0.32836, + "map_at_10": 0.47016, + "map_at_100": 0.48047, + "map_at_1000": 0.48125, + "map_at_20": 0.47602, + "map_at_3": 0.43508, + "map_at_5": 0.45587, + "mrr_at_1": 0.6567184334908845, + "mrr_at_10": 0.7374106406010503, + "mrr_at_100": 0.7407360674005218, + "mrr_at_1000": 0.7408528396466417, + "mrr_at_20": 0.7395171995882409, + "mrr_at_3": 0.7212469052442031, + "mrr_at_5": 0.7312401530497377, + "nauc_map_at_1000_diff1": 0.34005453539011166, + "nauc_map_at_1000_max": 0.2582115616984131, + "nauc_map_at_1000_std": 0.014158465225486873, + "nauc_map_at_100_diff1": 0.3397885841853144, + "nauc_map_at_100_max": 0.25788066937953535, + "nauc_map_at_100_std": 0.014014125260716204, + "nauc_map_at_10_diff1": 0.34208269490411625, + "nauc_map_at_10_max": 0.2581236166730827, + "nauc_map_at_10_std": 0.00812095823616366, + "nauc_map_at_1_diff1": 0.6954414023975188, + "nauc_map_at_1_max": 0.4065799250364921, + "nauc_map_at_1_std": -0.0503532335561708, + "nauc_map_at_20_diff1": 0.34042140247707, + "nauc_map_at_20_max": 0.25793308131593257, + "nauc_map_at_20_std": 0.011211505623511912, + "nauc_map_at_3_diff1": 0.37206053991747484, + "nauc_map_at_3_max": 0.27418797479842416, + "nauc_map_at_3_std": -0.0076400779985672605, + "nauc_map_at_5_diff1": 0.35018465182394753, + "nauc_map_at_5_max": 0.264335126786127, + "nauc_map_at_5_std": 0.0011297600218386353, + "nauc_mrr_at_1000_diff1": 0.6725936635050808, + "nauc_mrr_at_1000_max": 0.41714918673128437, + "nauc_mrr_at_1000_std": -0.03087319936519033, + "nauc_mrr_at_100_diff1": 0.6725590240269183, + "nauc_mrr_at_100_max": 0.4171790879024321, + "nauc_mrr_at_100_std": -0.030740353666702233, + "nauc_mrr_at_10_diff1": 0.671799911142862, + "nauc_mrr_at_10_max": 0.41714875398588824, + "nauc_mrr_at_10_std": -0.030806545113886787, + "nauc_mrr_at_1_diff1": 0.6954414023975188, + "nauc_mrr_at_1_max": 0.4065799250364921, + "nauc_mrr_at_1_std": -0.0503532335561708, + "nauc_mrr_at_20_diff1": 0.6722701732432218, + "nauc_mrr_at_20_max": 0.41723520291153915, + "nauc_mrr_at_20_std": -0.030830587852967353, + "nauc_mrr_at_3_diff1": 0.6696430614667667, + "nauc_mrr_at_3_max": 0.4148604737080645, + "nauc_mrr_at_3_std": -0.036481401224598475, + "nauc_mrr_at_5_diff1": 0.6718668927352158, + "nauc_mrr_at_5_max": 0.41710678256086414, + "nauc_mrr_at_5_std": -0.03428397229624744, + "nauc_ndcg_at_1000_diff1": 0.3707610468655133, + "nauc_ndcg_at_1000_max": 0.28224540769228235, + "nauc_ndcg_at_1000_std": 0.04185248548344529, + "nauc_ndcg_at_100_diff1": 0.3642380555672608, + "nauc_ndcg_at_100_max": 0.27489609981662105, + "nauc_ndcg_at_100_std": 0.04054837611491843, + "nauc_ndcg_at_10_diff1": 0.3747804056929861, + "nauc_ndcg_at_10_max": 0.27753495611587303, + "nauc_ndcg_at_10_std": 0.01719221993401221, + "nauc_ndcg_at_1_diff1": 0.6954414023975188, + "nauc_ndcg_at_1_max": 0.4065799250364921, + "nauc_ndcg_at_1_std": -0.0503532335561708, + "nauc_ndcg_at_20_diff1": 0.3684564644726093, + "nauc_ndcg_at_20_max": 0.27576226304248097, + "nauc_ndcg_at_20_std": 0.02494939399202034, + "nauc_ndcg_at_3_diff1": 0.422828223866538, + "nauc_ndcg_at_3_max": 0.30300252457229687, + "nauc_ndcg_at_3_std": -0.008620078353970814, + "nauc_ndcg_at_5_diff1": 0.3917361365092243, + "nauc_ndcg_at_5_max": 0.2890704231405507, + "nauc_ndcg_at_5_std": 0.0028376558066016454, + "nauc_precision_at_1000_diff1": -0.030942823392612357, + "nauc_precision_at_1000_max": 0.1228787390319326, + "nauc_precision_at_1000_std": 0.2592838801244994, + "nauc_precision_at_100_diff1": 0.03649374343629417, + "nauc_precision_at_100_max": 0.11075555507772332, + "nauc_precision_at_100_std": 0.183299768701232, + "nauc_precision_at_10_diff1": 0.1541518743864118, + "nauc_precision_at_10_max": 0.1652239778879397, + "nauc_precision_at_10_std": 0.06837276762823966, + "nauc_precision_at_1_diff1": 0.6954414023975188, + "nauc_precision_at_1_max": 0.4065799250364921, + "nauc_precision_at_1_std": -0.0503532335561708, + "nauc_precision_at_20_diff1": 0.11421479372333689, + "nauc_precision_at_20_max": 0.1465873574938081, + "nauc_precision_at_20_std": 0.09377346877954564, + "nauc_precision_at_3_diff1": 0.2928638916411982, + "nauc_precision_at_3_max": 0.2494345281552621, + "nauc_precision_at_3_std": 0.012358684556900704, + "nauc_precision_at_5_diff1": 0.22154044068448586, + "nauc_precision_at_5_max": 0.21268338920126156, + "nauc_precision_at_5_std": 0.03352963627463402, + "nauc_recall_at_1000_diff1": -0.030942823392611615, + "nauc_recall_at_1000_max": 0.12287873903193287, + "nauc_recall_at_1000_std": 0.2592838801245011, + "nauc_recall_at_100_diff1": 0.03649374343629381, + "nauc_recall_at_100_max": 0.11075555507772299, + "nauc_recall_at_100_std": 0.18329976870123144, + "nauc_recall_at_10_diff1": 0.1541518743864111, + "nauc_recall_at_10_max": 0.16522397788793902, + "nauc_recall_at_10_std": 0.06837276762824004, + "nauc_recall_at_1_diff1": 0.6954414023975188, + "nauc_recall_at_1_max": 0.4065799250364921, + "nauc_recall_at_1_std": -0.0503532335561708, + "nauc_recall_at_20_diff1": 0.11421479372333719, + "nauc_recall_at_20_max": 0.1465873574938076, + "nauc_recall_at_20_std": 0.09377346877954627, + "nauc_recall_at_3_diff1": 0.29286389164119786, + "nauc_recall_at_3_max": 0.24943452815526215, + "nauc_recall_at_3_std": 0.012358684556899974, + "nauc_recall_at_5_diff1": 0.221540440684486, + "nauc_recall_at_5_max": 0.21268338920126187, + "nauc_recall_at_5_std": 0.03352963627463428, + "ndcg_at_1": 0.65672, + "ndcg_at_10": 0.56936, + "ndcg_at_100": 0.60723, + "ndcg_at_1000": 0.6226, + "ndcg_at_20": 0.58472, + "ndcg_at_3": 0.51526, + "ndcg_at_5": 0.54378, + "precision_at_1": 0.65672, + "precision_at_10": 0.12105, + "precision_at_100": 0.01507, + "precision_at_1000": 0.00171, + "precision_at_20": 0.06547, + "precision_at_3": 0.32312, + "precision_at_5": 0.21648, + "recall_at_1": 0.32836, + "recall_at_10": 0.60527, + "recall_at_100": 0.75375, + "recall_at_1000": 0.85544, + "recall_at_20": 0.65469, + "recall_at_3": 0.48467, + "recall_at_5": 0.54119 + } + ] + }, + "task_name": "HotpotQA" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/ImdbClassification.json b/ai-sage__Giga-Embeddings-instruct/external/ImdbClassification.json new file mode 100644 index 000000000..fb717bd76 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "evaluation_time": 1056.6063878536224, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.9490639999999999, + "ap": 0.9250413995343256, + "ap_weighted": 0.9250413995343256, + "f1": 0.9490353518558736, + "f1_weighted": 0.9490353518558736, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9490639999999999, + "scores_per_experiment": [ + { + "accuracy": 0.94852, + "ap": 0.9345307065808177, + "ap_weighted": 0.9345307065808177, + "f1": 0.9484958811567885, + "f1_weighted": 0.9484958811567886 + }, + { + "accuracy": 0.94288, + "ap": 0.9338995909878682, + "ap_weighted": 0.9338995909878682, + "f1": 0.9427956487518234, + "f1_weighted": 0.9427956487518234 + }, + { + "accuracy": 0.95576, + "ap": 0.9300653854539791, + "ap_weighted": 0.9300653854539791, + "f1": 0.9557517192705485, + "f1_weighted": 0.9557517192705485 + }, + { + "accuracy": 0.95668, + "ap": 0.9347663029535197, + "ap_weighted": 0.9347663029535197, + "f1": 0.9566788465482967, + "f1_weighted": 0.9566788465482966 + }, + { + "accuracy": 0.95164, + "ap": 0.9307989870568374, + "ap_weighted": 0.9307989870568374, + "f1": 0.95163971208219, + "f1_weighted": 0.9516397120821899 + }, + { + "accuracy": 0.95272, + "ap": 0.9302064735837046, + "ap_weighted": 0.9302064735837046, + "f1": 0.9527196502010601, + "f1_weighted": 0.9527196502010601 + }, + { + "accuracy": 0.95464, + "ap": 0.9313407770056854, + "ap_weighted": 0.9313407770056854, + "f1": 0.9546380479118987, + "f1_weighted": 0.9546380479118988 + }, + { + "accuracy": 0.94044, + "ap": 0.9001979128627625, + "ap_weighted": 0.9001979128627625, + "f1": 0.9403496436444445, + "f1_weighted": 0.9403496436444444 + }, + { + "accuracy": 0.9436, + "ap": 0.905172744893395, + "ap_weighted": 0.905172744893395, + "f1": 0.9435245129664547, + "f1_weighted": 0.9435245129664545 + }, + { + "accuracy": 0.94376, + "ap": 0.919435113964687, + "ap_weighted": 0.919435113964687, + "f1": 0.9437598560252314, + "f1_weighted": 0.9437598560252314 + } + ] + } + ] + }, + "task_name": "ImdbClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/InappropriatenessClassification.json b/ai-sage__Giga-Embeddings-instruct/external/InappropriatenessClassification.json new file mode 100644 index 000000000..6d0d3bf26 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/InappropriatenessClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", "evaluation_time": 109.23797869682312, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.83290234375, "ap": 0.7750711273019213, "ap_weighted": 0.7750711273019213, "f1": 0.8242886423677278, "f1_weighted": 0.8242886423677278, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.83290234375, "scores_per_experiment": [{"accuracy": 0.84149609375, "ap": 0.7736859961854461, "ap_weighted": 0.7736859961854461, "f1": 0.8334293352872415, "f1_weighted": 0.8334293352872415}, {"accuracy": 0.83368359375, "ap": 0.7638794513081395, "ap_weighted": 0.7638794513081395, "f1": 0.8255754284117934, "f1_weighted": 0.8255754284117934}, {"accuracy": 0.84833203125, "ap": 0.7986662462757312, "ap_weighted": 0.7986662462757312, "f1": 0.8399427547553203, "f1_weighted": 0.8399427547553203}, {"accuracy": 0.81317578125, "ap": 0.7630947639049827, "ap_weighted": 0.7630947639049827, "f1": 0.8039642710374743, "f1_weighted": 0.8039642710374743}, {"accuracy": 0.81610546875, "ap": 0.7798060013745148, "ap_weighted": 0.7798060013745148, "f1": 0.805179140818284, "f1_weighted": 0.805179140818284}, {"accuracy": 0.83368359375, "ap": 0.7726653343023256, "ap_weighted": 0.7726653343023256, "f1": 0.8256326675457342, "f1_weighted": 0.8256326675457342}, {"accuracy": 0.8624921875, "ap": 0.8061846348947895, "ap_weighted": 0.8061846348947895, "f1": 0.854468732085917, "f1_weighted": 0.854468732085917}, {"accuracy": 0.83221875, "ap": 0.7582170758928571, "ap_weighted": 0.7582170758928571, "f1": 0.8238316613653046, "f1_weighted": 0.8238316613653046}, {"accuracy": 0.8371015625, "ap": 0.7920303520114942, "ap_weighted": 0.7920303520114942, "f1": 0.828129750529182, "f1_weighted": 0.828129750529182}, {"accuracy": 0.810734375, "ap": 0.742481416868932, "ap_weighted": 0.742481416868932, "f1": 0.8027326818410268, "f1_weighted": 0.8027326818410268}]}]}, "task_name": "InappropriatenessClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/KinopoiskClassification.json b/ai-sage__Giga-Embeddings-instruct/external/KinopoiskClassification.json new file mode 100644 index 000000000..afc53059b --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/KinopoiskClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", "evaluation_time": 119.80069732666016, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.7488666666666667, "f1": 0.726314130184491, "f1_weighted": 0.7263141301844909, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.7488666666666667, "scores_per_experiment": [{"accuracy": 0.7593333333333333, "f1": 0.742034446305042, "f1_weighted": 0.7420344463050419}, {"accuracy": 0.738, "f1": 0.7056153262604875, "f1_weighted": 0.7056153262604876}, {"accuracy": 0.7586666666666667, "f1": 0.7355771482191243, "f1_weighted": 0.7355771482191242}, {"accuracy": 0.7386666666666667, "f1": 0.7092601759743893, "f1_weighted": 0.7092601759743893}, {"accuracy": 0.7293333333333334, "f1": 0.7110132979731344, "f1_weighted": 0.7110132979731343}, {"accuracy": 0.7593333333333333, "f1": 0.7370940839026124, "f1_weighted": 0.7370940839026123}, {"accuracy": 0.736, "f1": 0.7119796910322566, "f1_weighted": 0.7119796910322566}, {"accuracy": 0.7573333333333333, "f1": 0.7373386553330862, "f1_weighted": 0.7373386553330862}, {"accuracy": 0.7566666666666667, "f1": 0.7382620389484612, "f1_weighted": 0.7382620389484612}, {"accuracy": 0.7553333333333333, "f1": 0.7349664378963153, "f1_weighted": 0.7349664378963154}]}]}, "task_name": "KinopoiskClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MIRACLReranking.json b/ai-sage__Giga-Embeddings-instruct/external/MIRACLReranking.json new file mode 100644 index 000000000..46ddebf45 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "task_name": "MIRACLReranking", + "mteb_version": "1.25.5", + "scores": { + "dev": [ + { + "NDCG@1(MIRACL)": 0.28709, + "NDCG@3(MIRACL)": 0.34412, + "NDCG@5(MIRACL)": 0.38075, + "NDCG@10(MIRACL)": 0.46567, + "NDCG@20(MIRACL)": 0.47194, + "NDCG@100(MIRACL)": 0.49574, + "NDCG@1000(MIRACL)": 0.49574, + "MAP@1(MIRACL)": 0.17828, + "MAP@3(MIRACL)": 0.27682, + "MAP@5(MIRACL)": 0.31055, + "MAP@10(MIRACL)": 0.33949, + "MAP@20(MIRACL)": 0.35627, + "MAP@100(MIRACL)": 0.36672, + "MAP@1000(MIRACL)": 0.36672, + "Recall@1(MIRACL)": 0.17828, + "Recall@3(MIRACL)": 0.36267, + "Recall@5(MIRACL)": 0.45999, + "Recall@10(MIRACL)": 0.57522, + "Recall@20(MIRACL)": 0.67722, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "P@1(MIRACL)": 0.28709, + "P@3(MIRACL)": 0.21839, + "P@5(MIRACL)": 0.17706, + "P@10(MIRACL)": 0.12061, + "P@20(MIRACL)": 0.07674, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "nAUC_NDCG@1_max(MIRACL)": 0.024526, + "nAUC_NDCG@1_std(MIRACL)": 0.032302, + "nAUC_NDCG@1_diff1(MIRACL)": 0.107881, + "nAUC_NDCG@3_max(MIRACL)": 0.026048, + "nAUC_NDCG@3_std(MIRACL)": 0.035972, + "nAUC_NDCG@3_diff1(MIRACL)": 0.133954, + "nAUC_NDCG@5_max(MIRACL)": 0.041066, + "nAUC_NDCG@5_std(MIRACL)": 0.050742, + "nAUC_NDCG@5_diff1(MIRACL)": 0.122242, + "nAUC_NDCG@10_max(MIRACL)": 0.078683, + "nAUC_NDCG@10_std(MIRACL)": 0.076005, + "nAUC_NDCG@10_diff1(MIRACL)": 0.103765, + "nAUC_NDCG@20_max(MIRACL)": 0.117336, + "nAUC_NDCG@20_std(MIRACL)": 0.107067, + "nAUC_NDCG@20_diff1(MIRACL)": 0.078703, + "nAUC_NDCG@100_max(MIRACL)": 0.162874, + "nAUC_NDCG@100_std(MIRACL)": 0.103445, + "nAUC_NDCG@100_diff1(MIRACL)": 0.054097, + "nAUC_NDCG@1000_max(MIRACL)": 0.162874, + "nAUC_NDCG@1000_std(MIRACL)": 0.103445, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.054097, + "nAUC_MAP@1_max(MIRACL)": -0.025252, + "nAUC_MAP@1_std(MIRACL)": -0.020847, + "nAUC_MAP@1_diff1(MIRACL)": 0.207535, + "nAUC_MAP@3_max(MIRACL)": -0.004599, + "nAUC_MAP@3_std(MIRACL)": 0.00968, + "nAUC_MAP@3_diff1(MIRACL)": 0.167991, + "nAUC_MAP@5_max(MIRACL)": 0.00877, + "nAUC_MAP@5_std(MIRACL)": 0.026916, + "nAUC_MAP@5_diff1(MIRACL)": 0.149256, + "nAUC_MAP@10_max(MIRACL)": 0.030351, + "nAUC_MAP@10_std(MIRACL)": 0.042845, + "nAUC_MAP@10_diff1(MIRACL)": 0.137207, + "nAUC_MAP@20_max(MIRACL)": 0.047092, + "nAUC_MAP@20_std(MIRACL)": 0.060561, + "nAUC_MAP@20_diff1(MIRACL)": 0.126648, + "nAUC_MAP@100_max(MIRACL)": 0.059568, + "nAUC_MAP@100_std(MIRACL)": 0.060512, + "nAUC_MAP@100_diff1(MIRACL)": 0.119067, + "nAUC_MAP@1000_max(MIRACL)": 0.059568, + "nAUC_MAP@1000_std(MIRACL)": 0.060512, + "nAUC_MAP@1000_diff1(MIRACL)": 0.119067, + "nAUC_Recall@1_max(MIRACL)": -0.025252, + "nAUC_Recall@1_std(MIRACL)": -0.020847, + "nAUC_Recall@1_diff1(MIRACL)": 0.207535, + "nAUC_Recall@3_max(MIRACL)": 0.008641, + "nAUC_Recall@3_std(MIRACL)": 0.016468, + "nAUC_Recall@3_diff1(MIRACL)": 0.152972, + "nAUC_Recall@5_max(MIRACL)": 0.041491, + "nAUC_Recall@5_std(MIRACL)": 0.055277, + "nAUC_Recall@5_diff1(MIRACL)": 0.114296, + "nAUC_Recall@10_max(MIRACL)": 0.1297, + "nAUC_Recall@10_std(MIRACL)": 0.118681, + "nAUC_Recall@10_diff1(MIRACL)": 0.059695, + "nAUC_Recall@20_max(MIRACL)": 0.254, + "nAUC_Recall@20_std(MIRACL)": 0.218824, + "nAUC_Recall@20_diff1(MIRACL)": -0.01627, + "nAUC_Recall@100_max(MIRACL)": 0.562873, + "nAUC_Recall@100_std(MIRACL)": 0.248764, + "nAUC_Recall@100_diff1(MIRACL)": -0.153821, + "nAUC_Recall@1000_max(MIRACL)": 0.562873, + "nAUC_Recall@1000_std(MIRACL)": 0.248764, + "nAUC_Recall@1000_diff1(MIRACL)": -0.153821, + "nAUC_P@1_max(MIRACL)": 0.024526, + "nAUC_P@1_std(MIRACL)": 0.032302, + "nAUC_P@1_diff1(MIRACL)": 0.107881, + "nAUC_P@3_max(MIRACL)": 0.081219, + "nAUC_P@3_std(MIRACL)": 0.102664, + "nAUC_P@3_diff1(MIRACL)": 0.009984, + "nAUC_P@5_max(MIRACL)": 0.129244, + "nAUC_P@5_std(MIRACL)": 0.130437, + "nAUC_P@5_diff1(MIRACL)": -0.046018, + "nAUC_P@10_max(MIRACL)": 0.192036, + "nAUC_P@10_std(MIRACL)": 0.160371, + "nAUC_P@10_diff1(MIRACL)": -0.088654, + "nAUC_P@20_max(MIRACL)": 0.242356, + "nAUC_P@20_std(MIRACL)": 0.193244, + "nAUC_P@20_diff1(MIRACL)": -0.137228, + "nAUC_P@100_max(MIRACL)": 0.281759, + "nAUC_P@100_std(MIRACL)": 0.125112, + "nAUC_P@100_diff1(MIRACL)": -0.183748, + "nAUC_P@1000_max(MIRACL)": 0.281759, + "nAUC_P@1000_std(MIRACL)": 0.125112, + "nAUC_P@1000_diff1(MIRACL)": -0.183748, + "main_score": 0.46567, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 2535.7574694156647, + "kg_co2_emissions": null +} diff --git a/ai-sage__Giga-Embeddings-instruct/external/MIRACLRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/MIRACLRetrieval.json new file mode 100644 index 000000000..b7bd61583 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 46533.344428777695, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.58919, + "map_at_1": 0.28105, + "map_at_10": 0.48485, + "map_at_100": 0.51092, + "map_at_1000": 0.51197, + "map_at_20": 0.49964, + "map_at_3": 0.40702, + "map_at_5": 0.44908, + "mrr_at_1": 0.536741214057508, + "mrr_at_10": 0.659469610527918, + "mrr_at_100": 0.6632118832568914, + "mrr_at_1000": 0.6632790976216316, + "mrr_at_20": 0.661976638975423, + "mrr_at_3": 0.6320553780617684, + "mrr_at_5": 0.6507454739084141, + "nauc_map_at_1000_diff1": 0.28692875448815813, + "nauc_map_at_1000_max": 0.1249177936928466, + "nauc_map_at_1000_std": -0.0262205081625825, + "nauc_map_at_100_diff1": 0.28645110165399174, + "nauc_map_at_100_max": 0.12456098789482797, + "nauc_map_at_100_std": -0.026613261048769606, + "nauc_map_at_10_diff1": 0.2847293754157533, + "nauc_map_at_10_max": 0.10810116535339406, + "nauc_map_at_10_std": -0.052865884745125755, + "nauc_map_at_1_diff1": 0.3284937124647936, + "nauc_map_at_1_max": 0.06422329814718059, + "nauc_map_at_1_std": -0.09338607410867883, + "nauc_map_at_20_diff1": 0.28599327426899934, + "nauc_map_at_20_max": 0.1182906430468277, + "nauc_map_at_20_std": -0.04021436740087691, + "nauc_map_at_3_diff1": 0.27482658420810435, + "nauc_map_at_3_max": 0.08017407826476448, + "nauc_map_at_3_std": -0.07867867276064061, + "nauc_map_at_5_diff1": 0.27800427415149925, + "nauc_map_at_5_max": 0.09039207813157996, + "nauc_map_at_5_std": -0.07591204490113271, + "nauc_mrr_at_1000_diff1": 0.35785782826995727, + "nauc_mrr_at_1000_max": 0.20786252125904436, + "nauc_mrr_at_1000_std": 0.08583860329566223, + "nauc_mrr_at_100_diff1": 0.35775930857323696, + "nauc_mrr_at_100_max": 0.20791530168252753, + "nauc_mrr_at_100_std": 0.0859657930717071, + "nauc_mrr_at_10_diff1": 0.3569769923988554, + "nauc_mrr_at_10_max": 0.20901225274243265, + "nauc_mrr_at_10_std": 0.08708683664351217, + "nauc_mrr_at_1_diff1": 0.3768512205937764, + "nauc_mrr_at_1_max": 0.17880526496865198, + "nauc_mrr_at_1_std": 0.04011086727439952, + "nauc_mrr_at_20_diff1": 0.3576194253028584, + "nauc_mrr_at_20_max": 0.208340188340251, + "nauc_mrr_at_20_std": 0.08681881702943871, + "nauc_mrr_at_3_diff1": 0.3565799054328497, + "nauc_mrr_at_3_max": 0.20406872858177758, + "nauc_mrr_at_3_std": 0.0796952304923041, + "nauc_mrr_at_5_diff1": 0.3600029293304689, + "nauc_mrr_at_5_max": 0.20893681203274397, + "nauc_mrr_at_5_std": 0.08608032961961529, + "nauc_ndcg_at_1000_diff1": 0.30283370916340396, + "nauc_ndcg_at_1000_max": 0.1745515207920199, + "nauc_ndcg_at_1000_std": 0.055008171225210276, + "nauc_ndcg_at_100_diff1": 0.2937087750574036, + "nauc_ndcg_at_100_max": 0.171659856137934, + "nauc_ndcg_at_100_std": 0.057406720302720114, + "nauc_ndcg_at_10_diff1": 0.2919089885399415, + "nauc_ndcg_at_10_max": 0.13581733885756453, + "nauc_ndcg_at_10_std": -0.011925279328006109, + "nauc_ndcg_at_1_diff1": 0.3768512205937764, + "nauc_ndcg_at_1_max": 0.17880526496865198, + "nauc_ndcg_at_1_std": 0.04011086727439952, + "nauc_ndcg_at_20_diff1": 0.2937764766181379, + "nauc_ndcg_at_20_max": 0.15345088185381453, + "nauc_ndcg_at_20_std": 0.015388051841648352, + "nauc_ndcg_at_3_diff1": 0.28550969354690575, + "nauc_ndcg_at_3_max": 0.13205099776372925, + "nauc_ndcg_at_3_std": 0.0012608790067765707, + "nauc_ndcg_at_5_diff1": 0.29262808940095303, + "nauc_ndcg_at_5_max": 0.1234844355941903, + "nauc_ndcg_at_5_std": -0.02503299305763401, + "nauc_precision_at_1000_diff1": -0.049469293142754264, + "nauc_precision_at_1000_max": 0.14318369622332805, + "nauc_precision_at_1000_std": 0.29953649623968, + "nauc_precision_at_100_diff1": -0.04875440708838266, + "nauc_precision_at_100_max": 0.16437239890966013, + "nauc_precision_at_100_std": 0.3286582107306089, + "nauc_precision_at_10_diff1": 0.0350201515428894, + "nauc_precision_at_10_max": 0.17756840808949528, + "nauc_precision_at_10_std": 0.22110419776166054, + "nauc_precision_at_1_diff1": 0.3768512205937764, + "nauc_precision_at_1_max": 0.17880526496865198, + "nauc_precision_at_1_std": 0.04011086727439952, + "nauc_precision_at_20_diff1": 0.0036248612466392234, + "nauc_precision_at_20_max": 0.18774584939318542, + "nauc_precision_at_20_std": 0.27061134898065725, + "nauc_precision_at_3_diff1": 0.12364445300291128, + "nauc_precision_at_3_max": 0.17198756066552692, + "nauc_precision_at_3_std": 0.13396863299435208, + "nauc_precision_at_5_diff1": 0.07903105219850089, + "nauc_precision_at_5_max": 0.1720630661570393, + "nauc_precision_at_5_std": 0.16411951051739743, + "nauc_recall_at_1000_diff1": 0.3898954496312062, + "nauc_recall_at_1000_max": 0.5533728064769203, + "nauc_recall_at_1000_std": 0.630550070852272, + "nauc_recall_at_100_diff1": 0.12772525292034662, + "nauc_recall_at_100_max": 0.21511664999914887, + "nauc_recall_at_100_std": 0.2855416139778685, + "nauc_recall_at_10_diff1": 0.21397118870496928, + "nauc_recall_at_10_max": 0.07632620412302538, + "nauc_recall_at_10_std": -0.05738894810245653, + "nauc_recall_at_1_diff1": 0.3284937124647936, + "nauc_recall_at_1_max": 0.06422329814718059, + "nauc_recall_at_1_std": -0.09338607410867883, + "nauc_recall_at_20_diff1": 0.1950218742436598, + "nauc_recall_at_20_max": 0.10660163096971174, + "nauc_recall_at_20_std": 0.009197930988253975, + "nauc_recall_at_3_diff1": 0.214039473712599, + "nauc_recall_at_3_max": 0.049820152592443066, + "nauc_recall_at_3_std": -0.08873734870059172, + "nauc_recall_at_5_diff1": 0.21487915917217768, + "nauc_recall_at_5_max": 0.05118393026118494, + "nauc_recall_at_5_std": -0.10131702930834897, + "ndcg_at_1": 0.53674, + "ndcg_at_10": 0.58919, + "ndcg_at_100": 0.65736, + "ndcg_at_1000": 0.66932, + "ndcg_at_20": 0.61972, + "ndcg_at_3": 0.5202, + "ndcg_at_5": 0.54631, + "precision_at_1": 0.53674, + "precision_at_10": 0.17332, + "precision_at_100": 0.02546, + "precision_at_1000": 0.00279, + "precision_at_20": 0.10152, + "precision_at_3": 0.34798, + "precision_at_5": 0.26805, + "recall_at_1": 0.28105, + "recall_at_10": 0.70024, + "recall_at_100": 0.91739, + "recall_at_1000": 0.98131, + "recall_at_20": 0.78385, + "recall_at_3": 0.48199, + "recall_at_5": 0.58495 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MSMARCO.json b/ai-sage__Giga-Embeddings-instruct/external/MSMARCO.json new file mode 100644 index 000000000..5cac68cc0 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MSMARCO.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "evaluation_time": 41928.7965323925, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.61953, + "map_at_1": 0.02295, + "map_at_10": 0.13425, + "map_at_100": 0.36994, + "map_at_1000": 0.45477, + "map_at_20": 0.20542, + "map_at_3": 0.05632, + "map_at_5": 0.07935, + "mrr_at_1": 0.8604651162790697, + "mrr_at_10": 0.90968992248062, + "mrr_at_100": 0.90968992248062, + "mrr_at_1000": 0.90968992248062, + "mrr_at_20": 0.90968992248062, + "mrr_at_3": 0.8992248062015503, + "mrr_at_5": 0.90968992248062, + "nauc_map_at_1000_diff1": -0.3011984354598437, + "nauc_map_at_1000_max": 0.4112162221478454, + "nauc_map_at_1000_std": 0.5394915916714401, + "nauc_map_at_100_diff1": -0.18474186072111015, + "nauc_map_at_100_max": 0.3255771055611539, + "nauc_map_at_100_std": 0.34850257861392364, + "nauc_map_at_10_diff1": 0.12626067523953438, + "nauc_map_at_10_max": 0.13292172274891448, + "nauc_map_at_10_std": -0.027683801353352255, + "nauc_map_at_1_diff1": 0.442949717144564, + "nauc_map_at_1_max": -0.053041256958258884, + "nauc_map_at_1_std": -0.2985748050243832, + "nauc_map_at_20_diff1": 0.016976308361799895, + "nauc_map_at_20_max": 0.18879635307706247, + "nauc_map_at_20_std": 0.06078598723546325, + "nauc_map_at_3_diff1": 0.36091324304299205, + "nauc_map_at_3_max": 0.028220999625196037, + "nauc_map_at_3_std": -0.21476895853466463, + "nauc_map_at_5_diff1": 0.275419170591421, + "nauc_map_at_5_max": 0.09290150393583366, + "nauc_map_at_5_std": -0.13742406800486986, + "nauc_mrr_at_1000_diff1": -0.076099403523558, + "nauc_mrr_at_1000_max": 0.7501353061667478, + "nauc_mrr_at_1000_std": 0.6872722608823327, + "nauc_mrr_at_100_diff1": -0.076099403523558, + "nauc_mrr_at_100_max": 0.7501353061667478, + "nauc_mrr_at_100_std": 0.6872722608823327, + "nauc_mrr_at_10_diff1": -0.076099403523558, + "nauc_mrr_at_10_max": 0.7501353061667478, + "nauc_mrr_at_10_std": 0.6872722608823327, + "nauc_mrr_at_1_diff1": -0.07227836484041839, + "nauc_mrr_at_1_max": 0.7007087438233301, + "nauc_mrr_at_1_std": 0.6228193431865652, + "nauc_mrr_at_20_diff1": -0.076099403523558, + "nauc_mrr_at_20_max": 0.7501353061667478, + "nauc_mrr_at_20_std": 0.6872722608823327, + "nauc_mrr_at_3_diff1": -0.02664317491139411, + "nauc_mrr_at_3_max": 0.7691690813390306, + "nauc_mrr_at_3_std": 0.7041867051808182, + "nauc_mrr_at_5_diff1": -0.076099403523558, + "nauc_mrr_at_5_max": 0.7501353061667478, + "nauc_mrr_at_5_std": 0.6872722608823327, + "nauc_ndcg_at_1000_diff1": -0.34588209720849367, + "nauc_ndcg_at_1000_max": 0.5401529717339395, + "nauc_ndcg_at_1000_std": 0.5931637471252265, + "nauc_ndcg_at_100_diff1": -0.2664028813352152, + "nauc_ndcg_at_100_max": 0.35134075532155673, + "nauc_ndcg_at_100_std": 0.4790610288563949, + "nauc_ndcg_at_10_diff1": -0.3016559486015698, + "nauc_ndcg_at_10_max": 0.38910360318642523, + "nauc_ndcg_at_10_std": 0.4268661569167478, + "nauc_ndcg_at_1_diff1": -0.015593104420333766, + "nauc_ndcg_at_1_max": 0.4261039841149629, + "nauc_ndcg_at_1_std": 0.35840800888923036, + "nauc_ndcg_at_20_diff1": -0.3077391764501866, + "nauc_ndcg_at_20_max": 0.3450283605663452, + "nauc_ndcg_at_20_std": 0.3759483221610664, + "nauc_ndcg_at_3_diff1": -0.15520713715926346, + "nauc_ndcg_at_3_max": 0.4578611982790275, + "nauc_ndcg_at_3_std": 0.4597839281793152, + "nauc_ndcg_at_5_diff1": -0.2385285278088875, + "nauc_ndcg_at_5_max": 0.4439780563283533, + "nauc_ndcg_at_5_std": 0.4239949056653093, + "nauc_precision_at_1000_diff1": -0.30417558924768134, + "nauc_precision_at_1000_max": 0.13548159537058152, + "nauc_precision_at_1000_std": 0.3725751589837348, + "nauc_precision_at_100_diff1": -0.3646125649080512, + "nauc_precision_at_100_max": 0.20741866361147507, + "nauc_precision_at_100_std": 0.47312413368139156, + "nauc_precision_at_10_diff1": -0.5904379077622773, + "nauc_precision_at_10_max": 0.4666289912456372, + "nauc_precision_at_10_std": 0.6276962343960867, + "nauc_precision_at_1_diff1": -0.07227836484041839, + "nauc_precision_at_1_max": 0.7007087438233301, + "nauc_precision_at_1_std": 0.6228193431865652, + "nauc_precision_at_20_diff1": -0.5310996336661844, + "nauc_precision_at_20_max": 0.3809482517069248, + "nauc_precision_at_20_std": 0.5673611578181776, + "nauc_precision_at_3_diff1": -0.36999469625002207, + "nauc_precision_at_3_max": 0.6412921150590218, + "nauc_precision_at_3_std": 0.6478671082210092, + "nauc_precision_at_5_diff1": -0.46500845210255337, + "nauc_precision_at_5_max": 0.594191275770431, + "nauc_precision_at_5_std": 0.6800855518751847, + "nauc_recall_at_1000_diff1": -0.24555235403948775, + "nauc_recall_at_1000_max": 0.5422435135708867, + "nauc_recall_at_1000_std": 0.4931140174115312, + "nauc_recall_at_100_diff1": -0.0370014715108285, + "nauc_recall_at_100_max": 0.1921098260931646, + "nauc_recall_at_100_std": 0.18131361156635536, + "nauc_recall_at_10_diff1": 0.1609333751982658, + "nauc_recall_at_10_max": 0.07004596323208447, + "nauc_recall_at_10_std": -0.09042361327667667, + "nauc_recall_at_1_diff1": 0.442949717144564, + "nauc_recall_at_1_max": -0.053041256958258884, + "nauc_recall_at_1_std": -0.2985748050243832, + "nauc_recall_at_20_diff1": 0.09048625900019916, + "nauc_recall_at_20_max": 0.10488512958255308, + "nauc_recall_at_20_std": -0.03079039962366688, + "nauc_recall_at_3_diff1": 0.36372956859064703, + "nauc_recall_at_3_max": 0.016785799970091962, + "nauc_recall_at_3_std": -0.22297408897380536, + "nauc_recall_at_5_diff1": 0.2740492518632176, + "nauc_recall_at_5_max": 0.04594462203289419, + "nauc_recall_at_5_std": -0.17585190330631453, + "ndcg_at_1": 0.66279, + "ndcg_at_10": 0.61953, + "ndcg_at_100": 0.61384, + "ndcg_at_1000": 0.71183, + "ndcg_at_20": 0.61391, + "ndcg_at_3": 0.63244, + "ndcg_at_5": 0.61757, + "precision_at_1": 0.86047, + "precision_at_10": 0.73256, + "precision_at_100": 0.37535, + "precision_at_1000": 0.07579, + "precision_at_20": 0.65349, + "precision_at_3": 0.79845, + "precision_at_5": 0.75814, + "recall_at_1": 0.02295, + "recall_at_10": 0.15101, + "recall_at_100": 0.51615, + "recall_at_1000": 0.82471, + "recall_at_20": 0.24096, + "recall_at_3": 0.05773, + "recall_at_5": 0.085 + } + ] + }, + "task_name": "MSMARCO" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MTOPDomainClassification.json b/ai-sage__Giga-Embeddings-instruct/external/MTOPDomainClassification.json new file mode 100644 index 000000000..c83ce8a1f --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "evaluation_time": 104.06943845748901, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.9535111719106247, + "f1": 0.9518886239707826, + "f1_weighted": 0.9535614308126868, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9535111719106247, + "scores_per_experiment": [ + { + "accuracy": 0.9534883720930233, + "f1": 0.9513068314539946, + "f1_weighted": 0.953860982740785 + }, + { + "accuracy": 0.9582763337893296, + "f1": 0.956487828202021, + "f1_weighted": 0.958365924378379 + }, + { + "accuracy": 0.9514363885088919, + "f1": 0.9500814633642782, + "f1_weighted": 0.9513846252295896 + }, + { + "accuracy": 0.9596443228454172, + "f1": 0.9586843234131599, + "f1_weighted": 0.959831248746165 + }, + { + "accuracy": 0.9523483812129503, + "f1": 0.9487520798191647, + "f1_weighted": 0.9524336527442928 + }, + { + "accuracy": 0.9457364341085271, + "f1": 0.9458848665839118, + "f1_weighted": 0.9455340051339257 + }, + { + "accuracy": 0.9569083447332422, + "f1": 0.9552447107370887, + "f1_weighted": 0.9566375577687185 + }, + { + "accuracy": 0.9532603739170087, + "f1": 0.9500312272718439, + "f1_weighted": 0.9533344050351013 + }, + { + "accuracy": 0.9464204286365709, + "f1": 0.946796580336859, + "f1_weighted": 0.9465784062196791 + }, + { + "accuracy": 0.957592339261286, + "f1": 0.9556163285255022, + "f1_weighted": 0.9576535001302336 + } + ] + } + ] + }, + "task_name": "MTOPDomainClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MTOPIntentClassification.json b/ai-sage__Giga-Embeddings-instruct/external/MTOPIntentClassification.json new file mode 100644 index 000000000..df3605d2e --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MTOPIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", + "evaluation_time": 127.81569290161133, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.8047423620611035, + "f1": 0.5899139430808884, + "f1_weighted": 0.8226823539179561, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8047423620611035, + "scores_per_experiment": [ + { + "accuracy": 0.779297765617875, + "f1": 0.5784699895454685, + "f1_weighted": 0.8034387962345845 + }, + { + "accuracy": 0.8098495212038304, + "f1": 0.5842041328890061, + "f1_weighted": 0.8290024935664206 + }, + { + "accuracy": 0.8399452804377565, + "f1": 0.6119771167266209, + "f1_weighted": 0.8519034965870772 + }, + { + "accuracy": 0.8260373917008664, + "f1": 0.5997846739653878, + "f1_weighted": 0.8452515479229001 + }, + { + "accuracy": 0.8000455996352029, + "f1": 0.5875719406642353, + "f1_weighted": 0.8165280784076128 + }, + { + "accuracy": 0.8000455996352029, + "f1": 0.5902486644495873, + "f1_weighted": 0.81502627141091 + }, + { + "accuracy": 0.8144094847241222, + "f1": 0.5891804820482137, + "f1_weighted": 0.8356948651862466 + }, + { + "accuracy": 0.7756497948016415, + "f1": 0.5767803064759328, + "f1_weighted": 0.7957901785756146 + }, + { + "accuracy": 0.8075695394436845, + "f1": 0.6059194710839396, + "f1_weighted": 0.8279311967760784 + }, + { + "accuracy": 0.7945736434108527, + "f1": 0.5750026529604911, + "f1_weighted": 0.8062566145121165 + } + ] + } + ] + }, + "task_name": "MTOPIntentClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MassiveIntentClassification.json b/ai-sage__Giga-Embeddings-instruct/external/MassiveIntentClassification.json new file mode 100644 index 000000000..0f9c7d228 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MassiveIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 120.45010733604431, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.7515467383994621, + "f1": 0.7280821138956516, + "f1_weighted": 0.7406042145052858, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7515467383994621, + "scores_per_experiment": [ + { + "accuracy": 0.7700067249495629, + "f1": 0.7436278992049412, + "f1_weighted": 0.7643602802138477 + }, + { + "accuracy": 0.7689979825151312, + "f1": 0.7471422116409301, + "f1_weighted": 0.7607121739246011 + }, + { + "accuracy": 0.7451244115669132, + "f1": 0.721706395660248, + "f1_weighted": 0.7308259988514979 + }, + { + "accuracy": 0.7585743106926698, + "f1": 0.7235750676175341, + "f1_weighted": 0.745994296054399 + }, + { + "accuracy": 0.7427706792199058, + "f1": 0.7186086679258736, + "f1_weighted": 0.7249206829653466 + }, + { + "accuracy": 0.7212508406186954, + "f1": 0.7169223865142272, + "f1_weighted": 0.7103840580334434 + }, + { + "accuracy": 0.7454606590450572, + "f1": 0.7233124191926091, + "f1_weighted": 0.7360281282560626 + }, + { + "accuracy": 0.7518493611297915, + "f1": 0.726404417640471, + "f1_weighted": 0.739024707808752 + }, + { + "accuracy": 0.7488231338264963, + "f1": 0.7268821072115659, + "f1_weighted": 0.7378731731619954 + }, + { + "accuracy": 0.7626092804303968, + "f1": 0.732639566348116, + "f1_weighted": 0.7559186457829131 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/MassiveScenarioClassification.json b/ai-sage__Giga-Embeddings-instruct/external/MassiveScenarioClassification.json new file mode 100644 index 000000000..5a07ef535 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 219.10689449310303, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.7817417619367855, + "f1": 0.7700893008191851, + "f1_weighted": 0.7794617575337399, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7817417619367855, + "scores_per_experiment": [ + { + "accuracy": 0.8066577000672495, + "f1": 0.7981110376394867, + "f1_weighted": 0.8057415707923841 + }, + { + "accuracy": 0.785137861466039, + "f1": 0.7692046514445465, + "f1_weighted": 0.7833517883939476 + }, + { + "accuracy": 0.7834566240753195, + "f1": 0.7666986926750066, + "f1_weighted": 0.7820449950379882 + }, + { + "accuracy": 0.7881640887693342, + "f1": 0.7726314755092157, + "f1_weighted": 0.7843135488971323 + }, + { + "accuracy": 0.7679892400806994, + "f1": 0.7599799176889461, + "f1_weighted": 0.7627722639172921 + }, + { + "accuracy": 0.7659717552118359, + "f1": 0.7537908536917777, + "f1_weighted": 0.7628381746736386 + }, + { + "accuracy": 0.777404169468729, + "f1": 0.7621482146104179, + "f1_weighted": 0.7761116938173679 + }, + { + "accuracy": 0.7686617350369872, + "f1": 0.7656711266473281, + "f1_weighted": 0.7691534821881301 + }, + { + "accuracy": 0.7784129119031608, + "f1": 0.7717565949084322, + "f1_weighted": 0.7750284822369596 + }, + { + "accuracy": 0.7955615332885003, + "f1": 0.7809004433766941, + "f1_weighted": 0.7932615753825587 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.778160354156419, + "f1": 0.7667169331969356, + "f1_weighted": 0.7754451044923486, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.778160354156419, + "scores_per_experiment": [ + { + "accuracy": 0.8111165764879489, + "f1": 0.8038036934395777, + "f1_weighted": 0.8097832589774352 + }, + { + "accuracy": 0.7865223807181505, + "f1": 0.7707649117193927, + "f1_weighted": 0.7852860111575595 + }, + { + "accuracy": 0.7825873093949828, + "f1": 0.7643602496464034, + "f1_weighted": 0.7807497790994153 + }, + { + "accuracy": 0.7712739793408756, + "f1": 0.7559972240067983, + "f1_weighted": 0.7674971333252144 + }, + { + "accuracy": 0.7683226758484998, + "f1": 0.759201056405145, + "f1_weighted": 0.764063106876111 + }, + { + "accuracy": 0.7535661583866208, + "f1": 0.7427837143146001, + "f1_weighted": 0.7483699857039731 + }, + { + "accuracy": 0.7732415150024594, + "f1": 0.7562579687827958, + "f1_weighted": 0.7720710135878749 + }, + { + "accuracy": 0.7594687653713723, + "f1": 0.7551193713655037, + "f1_weighted": 0.7597177768396045 + }, + { + "accuracy": 0.7820954254795868, + "f1": 0.7801941585110552, + "f1_weighted": 0.7774306864331869 + }, + { + "accuracy": 0.793408755533694, + "f1": 0.7786869837780847, + "f1_weighted": 0.7894822929231115 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/NFCorpus.json b/ai-sage__Giga-Embeddings-instruct/external/NFCorpus.json new file mode 100644 index 000000000..45dbe2057 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "evaluation_time": 78.19622921943665, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.36029, + "map_at_1": 0.05932, + "map_at_10": 0.13353, + "map_at_100": 0.17, + "map_at_1000": 0.18565, + "map_at_20": 0.14803, + "map_at_3": 0.0983, + "map_at_5": 0.11472, + "mrr_at_1": 0.4674922600619195, + "mrr_at_10": 0.5679627991547497, + "mrr_at_100": 0.5737706490767912, + "mrr_at_1000": 0.5741967573448374, + "mrr_at_20": 0.571211287532972, + "mrr_at_3": 0.5495356037151704, + "mrr_at_5": 0.5606811145510837, + "nauc_map_at_1000_diff1": 0.35301138973780677, + "nauc_map_at_1000_max": 0.34424969167805747, + "nauc_map_at_1000_std": 0.20929139100817634, + "nauc_map_at_100_diff1": 0.3611891724859925, + "nauc_map_at_100_max": 0.3442849138137279, + "nauc_map_at_100_std": 0.19113502961983989, + "nauc_map_at_10_diff1": 0.4104900439765621, + "nauc_map_at_10_max": 0.3102146168233007, + "nauc_map_at_10_std": 0.09498859153190814, + "nauc_map_at_1_diff1": 0.511222877824682, + "nauc_map_at_1_max": 0.18489074115470763, + "nauc_map_at_1_std": -0.02614775958130393, + "nauc_map_at_20_diff1": 0.38199531357069133, + "nauc_map_at_20_max": 0.33106279037350905, + "nauc_map_at_20_std": 0.13927078976412968, + "nauc_map_at_3_diff1": 0.4742097675315478, + "nauc_map_at_3_max": 0.265657921989551, + "nauc_map_at_3_std": 0.041891355270209665, + "nauc_map_at_5_diff1": 0.4471175314512709, + "nauc_map_at_5_max": 0.2888308813297792, + "nauc_map_at_5_std": 0.05582422026464234, + "nauc_mrr_at_1000_diff1": 0.33713547810026767, + "nauc_mrr_at_1000_max": 0.43726417691590785, + "nauc_mrr_at_1000_std": 0.2754102562853434, + "nauc_mrr_at_100_diff1": 0.33695977574098473, + "nauc_mrr_at_100_max": 0.43748094067502613, + "nauc_mrr_at_100_std": 0.27584428643679404, + "nauc_mrr_at_10_diff1": 0.33831498127288695, + "nauc_mrr_at_10_max": 0.4360703620245724, + "nauc_mrr_at_10_std": 0.2724838397886887, + "nauc_mrr_at_1_diff1": 0.3134357704107609, + "nauc_mrr_at_1_max": 0.39593265875570277, + "nauc_mrr_at_1_std": 0.21334421193946485, + "nauc_mrr_at_20_diff1": 0.33766553533285454, + "nauc_mrr_at_20_max": 0.43605486908046387, + "nauc_mrr_at_20_std": 0.2727761848520149, + "nauc_mrr_at_3_diff1": 0.34498566125110586, + "nauc_mrr_at_3_max": 0.43051850338353187, + "nauc_mrr_at_3_std": 0.2658778632911902, + "nauc_mrr_at_5_diff1": 0.3428107082592715, + "nauc_mrr_at_5_max": 0.43777595430901617, + "nauc_mrr_at_5_std": 0.2751020660226881, + "nauc_ndcg_at_1000_diff1": 0.34024168682058387, + "nauc_ndcg_at_1000_max": 0.45616471969818406, + "nauc_ndcg_at_1000_std": 0.35739392736461173, + "nauc_ndcg_at_100_diff1": 0.317510374153008, + "nauc_ndcg_at_100_max": 0.3995194747369657, + "nauc_ndcg_at_100_std": 0.29099636537256385, + "nauc_ndcg_at_10_diff1": 0.25878621628641924, + "nauc_ndcg_at_10_max": 0.3749225635027015, + "nauc_ndcg_at_10_std": 0.2529534976285755, + "nauc_ndcg_at_1_diff1": 0.31625392306396943, + "nauc_ndcg_at_1_max": 0.39044106231127823, + "nauc_ndcg_at_1_std": 0.22316304086718933, + "nauc_ndcg_at_20_diff1": 0.27062451828760237, + "nauc_ndcg_at_20_max": 0.3699168550915193, + "nauc_ndcg_at_20_std": 0.2615811674137553, + "nauc_ndcg_at_3_diff1": 0.2607107932833021, + "nauc_ndcg_at_3_max": 0.38047345878557054, + "nauc_ndcg_at_3_std": 0.22940589033430345, + "nauc_ndcg_at_5_diff1": 0.2597420963086261, + "nauc_ndcg_at_5_max": 0.390151165636691, + "nauc_ndcg_at_5_std": 0.2472377772080157, + "nauc_precision_at_1000_diff1": -0.1102371885050686, + "nauc_precision_at_1000_max": -0.050681543757818276, + "nauc_precision_at_1000_std": 0.1185905313652616, + "nauc_precision_at_100_diff1": -0.0827923233471529, + "nauc_precision_at_100_max": 0.09303415360666892, + "nauc_precision_at_100_std": 0.26935679867455153, + "nauc_precision_at_10_diff1": 0.03558842040982842, + "nauc_precision_at_10_max": 0.3062292167383454, + "nauc_precision_at_10_std": 0.30831543521961724, + "nauc_precision_at_1_diff1": 0.3218214923067921, + "nauc_precision_at_1_max": 0.4003416082124917, + "nauc_precision_at_1_std": 0.2194136201407572, + "nauc_precision_at_20_diff1": -0.022160081796886288, + "nauc_precision_at_20_max": 0.23696772836984598, + "nauc_precision_at_20_std": 0.32325015781351785, + "nauc_precision_at_3_diff1": 0.17327428569744058, + "nauc_precision_at_3_max": 0.3696631025994581, + "nauc_precision_at_3_std": 0.2540310439934565, + "nauc_precision_at_5_diff1": 0.11325073410234283, + "nauc_precision_at_5_max": 0.3527087123202698, + "nauc_precision_at_5_std": 0.2743381915118116, + "nauc_recall_at_1000_diff1": 0.19343460354273634, + "nauc_recall_at_1000_max": 0.21188913677267576, + "nauc_recall_at_1000_std": 0.1875561707338635, + "nauc_recall_at_100_diff1": 0.2354227828659756, + "nauc_recall_at_100_max": 0.25795529811599555, + "nauc_recall_at_100_std": 0.1920436682986688, + "nauc_recall_at_10_diff1": 0.3657350490861877, + "nauc_recall_at_10_max": 0.2698041459230936, + "nauc_recall_at_10_std": 0.06448693265410069, + "nauc_recall_at_1_diff1": 0.511222877824682, + "nauc_recall_at_1_max": 0.18489074115470763, + "nauc_recall_at_1_std": -0.02614775958130393, + "nauc_recall_at_20_diff1": 0.31934925086533644, + "nauc_recall_at_20_max": 0.28186182157743717, + "nauc_recall_at_20_std": 0.10990757869684473, + "nauc_recall_at_3_diff1": 0.47315286481424645, + "nauc_recall_at_3_max": 0.2636699566948872, + "nauc_recall_at_3_std": 0.041624834101340255, + "nauc_recall_at_5_diff1": 0.41523648162451543, + "nauc_recall_at_5_max": 0.27757237222978054, + "nauc_recall_at_5_std": 0.044771144935421676, + "ndcg_at_1": 0.44892, + "ndcg_at_10": 0.36029, + "ndcg_at_100": 0.3334, + "ndcg_at_1000": 0.42584, + "ndcg_at_20": 0.33653, + "ndcg_at_3": 0.41747, + "ndcg_at_5": 0.39204, + "precision_at_1": 0.4644, + "precision_at_10": 0.26625, + "precision_at_100": 0.0844, + "precision_at_1000": 0.02168, + "precision_at_20": 0.19737, + "precision_at_3": 0.39732, + "precision_at_5": 0.33994, + "recall_at_1": 0.05932, + "recall_at_10": 0.17523, + "recall_at_100": 0.34213, + "recall_at_1000": 0.67244, + "recall_at_20": 0.21367, + "recall_at_3": 0.10953, + "recall_at_5": 0.13676 + } + ] + }, + "task_name": "NFCorpus" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/NQ.json b/ai-sage__Giga-Embeddings-instruct/external/NQ.json new file mode 100644 index 000000000..a870452ef --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "evaluation_time": 16431.243108272552, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47417, + "map_at_1": 0.25089, + "map_at_10": 0.39582, + "map_at_100": 0.40891, + "map_at_1000": 0.4093, + "map_at_20": 0.40451, + "map_at_3": 0.34998, + "map_at_5": 0.37604, + "mrr_at_1": 0.28418308227114714, + "mrr_at_10": 0.4205473247622723, + "mrr_at_100": 0.43080018588766383, + "mrr_at_1000": 0.43106719179445246, + "mrr_at_20": 0.42759323084610057, + "mrr_at_3": 0.38156624179219734, + "mrr_at_5": 0.4042487446890695, + "nauc_map_at_1000_diff1": 0.2891528249267823, + "nauc_map_at_1000_max": 0.12793914414537005, + "nauc_map_at_1000_std": -0.010929507554964479, + "nauc_map_at_100_diff1": 0.2891080883187314, + "nauc_map_at_100_max": 0.12819170484926945, + "nauc_map_at_100_std": -0.010715029364921733, + "nauc_map_at_10_diff1": 0.2878480452256528, + "nauc_map_at_10_max": 0.12576414045375905, + "nauc_map_at_10_std": -0.015015042135306234, + "nauc_map_at_1_diff1": 0.3303005074446969, + "nauc_map_at_1_max": 0.08594681580083179, + "nauc_map_at_1_std": -0.04299975654790197, + "nauc_map_at_20_diff1": 0.28914182108026243, + "nauc_map_at_20_max": 0.1280967068305672, + "nauc_map_at_20_std": -0.011939469617180539, + "nauc_map_at_3_diff1": 0.28991050451790973, + "nauc_map_at_3_max": 0.11180438925592628, + "nauc_map_at_3_std": -0.033146760775266466, + "nauc_map_at_5_diff1": 0.28758243799705513, + "nauc_map_at_5_max": 0.120624973622888, + "nauc_map_at_5_std": -0.022694243325484503, + "nauc_mrr_at_1000_diff1": 0.2827168423988733, + "nauc_mrr_at_1000_max": 0.13928637140455785, + "nauc_mrr_at_1000_std": 0.012672195165656992, + "nauc_mrr_at_100_diff1": 0.28270359379156185, + "nauc_mrr_at_100_max": 0.13951816084768742, + "nauc_mrr_at_100_std": 0.012938704185608484, + "nauc_mrr_at_10_diff1": 0.280034727497709, + "nauc_mrr_at_10_max": 0.13931095921919853, + "nauc_mrr_at_10_std": 0.011269493369630486, + "nauc_mrr_at_1_diff1": 0.32445281839130014, + "nauc_mrr_at_1_max": 0.10425874491501147, + "nauc_mrr_at_1_std": -0.01223877452463658, + "nauc_mrr_at_20_diff1": 0.2823816805223673, + "nauc_mrr_at_20_max": 0.14021546198308193, + "nauc_mrr_at_20_std": 0.013081243309456347, + "nauc_mrr_at_3_diff1": 0.28067272464887294, + "nauc_mrr_at_3_max": 0.12695913784346569, + "nauc_mrr_at_3_std": -0.001855041855408822, + "nauc_mrr_at_5_diff1": 0.2774570897935483, + "nauc_mrr_at_5_max": 0.13699813791475782, + "nauc_mrr_at_5_std": 0.008500026849272696, + "nauc_ndcg_at_1000_diff1": 0.2824687682394225, + "nauc_ndcg_at_1000_max": 0.14847726899183505, + "nauc_ndcg_at_1000_std": 0.016167349963007876, + "nauc_ndcg_at_100_diff1": 0.2814488488771258, + "nauc_ndcg_at_100_max": 0.15618743122645243, + "nauc_ndcg_at_100_std": 0.024557213112065802, + "nauc_ndcg_at_10_diff1": 0.2744146440443296, + "nauc_ndcg_at_10_max": 0.148506532012367, + "nauc_ndcg_at_10_std": 0.008782842057690032, + "nauc_ndcg_at_1_diff1": 0.3254713711215239, + "nauc_ndcg_at_1_max": 0.10478653863508298, + "nauc_ndcg_at_1_std": -0.011496114859411906, + "nauc_ndcg_at_20_diff1": 0.28045210967473694, + "nauc_ndcg_at_20_max": 0.15644714013409133, + "nauc_ndcg_at_20_std": 0.019062527641824572, + "nauc_ndcg_at_3_diff1": 0.27668700218726017, + "nauc_ndcg_at_3_max": 0.12209702851489775, + "nauc_ndcg_at_3_std": -0.02322334570511838, + "nauc_ndcg_at_5_diff1": 0.2716752269329571, + "nauc_ndcg_at_5_max": 0.13840657657672922, + "nauc_ndcg_at_5_std": -0.005167877238881974, + "nauc_precision_at_1000_diff1": -0.06027834334076069, + "nauc_precision_at_1000_max": 0.11225657752379692, + "nauc_precision_at_1000_std": 0.19122936171005317, + "nauc_precision_at_100_diff1": -0.0037782279838820083, + "nauc_precision_at_100_max": 0.1872448662528021, + "nauc_precision_at_100_std": 0.23659665020224618, + "nauc_precision_at_10_diff1": 0.12958694637851698, + "nauc_precision_at_10_max": 0.193564028554887, + "nauc_precision_at_10_std": 0.12512745030359101, + "nauc_precision_at_1_diff1": 0.3254713711215239, + "nauc_precision_at_1_max": 0.10478653863508298, + "nauc_precision_at_1_std": -0.011496114859411906, + "nauc_precision_at_20_diff1": 0.09149579881076679, + "nauc_precision_at_20_max": 0.21207738882353103, + "nauc_precision_at_20_std": 0.1832152485970381, + "nauc_precision_at_3_diff1": 0.210372683571967, + "nauc_precision_at_3_max": 0.15684100593309513, + "nauc_precision_at_3_std": 0.024580907891125502, + "nauc_precision_at_5_diff1": 0.17206173306954048, + "nauc_precision_at_5_max": 0.18211885404364733, + "nauc_precision_at_5_std": 0.07436932427752223, + "nauc_recall_at_1000_diff1": 0.40285804045190193, + "nauc_recall_at_1000_max": 0.6050252714882125, + "nauc_recall_at_1000_std": 0.6526984741399422, + "nauc_recall_at_100_diff1": 0.2651711102966866, + "nauc_recall_at_100_max": 0.43373920041376357, + "nauc_recall_at_100_std": 0.3639302655748604, + "nauc_recall_at_10_diff1": 0.2285248270157389, + "nauc_recall_at_10_max": 0.20595760178488481, + "nauc_recall_at_10_std": 0.060722235537064496, + "nauc_recall_at_1_diff1": 0.3303005074446969, + "nauc_recall_at_1_max": 0.08594681580083179, + "nauc_recall_at_1_std": -0.04299975654790197, + "nauc_recall_at_20_diff1": 0.2524929241606468, + "nauc_recall_at_20_max": 0.2701732868361449, + "nauc_recall_at_20_std": 0.1303723631965964, + "nauc_recall_at_3_diff1": 0.24079954960566521, + "nauc_recall_at_3_max": 0.13160951322480183, + "nauc_recall_at_3_std": -0.024282680235922994, + "nauc_recall_at_5_diff1": 0.2248576829416849, + "nauc_recall_at_5_max": 0.16825953985618441, + "nauc_recall_at_5_std": 0.015133407674177162, + "ndcg_at_1": 0.28389, + "ndcg_at_10": 0.47417, + "ndcg_at_100": 0.52904, + "ndcg_at_1000": 0.53754, + "ndcg_at_20": 0.50282, + "ndcg_at_3": 0.38682, + "ndcg_at_5": 0.43059, + "precision_at_1": 0.28389, + "precision_at_10": 0.08239, + "precision_at_100": 0.01124, + "precision_at_1000": 0.0012, + "precision_at_20": 0.04783, + "precision_at_3": 0.17922, + "precision_at_5": 0.13285, + "recall_at_1": 0.25089, + "recall_at_10": 0.68931, + "recall_at_100": 0.92543, + "recall_at_1000": 0.98774, + "recall_at_20": 0.79633, + "recall_at_3": 0.46193, + "recall_at_5": 0.56257 + } + ] + }, + "task_name": "NQ" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/QuoraRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/QuoraRetrieval.json new file mode 100644 index 000000000..f686ae91b --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/QuoraRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "evaluation_time": 1043.8398144245148, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89237, + "map_at_1": 0.71553, + "map_at_10": 0.8559, + "map_at_100": 0.86205, + "map_at_1000": 0.86219, + "map_at_20": 0.85985, + "map_at_3": 0.82654, + "map_at_5": 0.84514, + "mrr_at_1": 0.8207, + "mrr_at_10": 0.8833148015872988, + "mrr_at_100": 0.884073911150083, + "mrr_at_1000": 0.8840805247679367, + "mrr_at_20": 0.8838813608158238, + "mrr_at_3": 0.8742999999999971, + "mrr_at_5": 0.8806649999999964, + "nauc_map_at_1000_diff1": 0.7708553781000765, + "nauc_map_at_1000_max": 0.23450946013228485, + "nauc_map_at_1000_std": -0.3527077466937614, + "nauc_map_at_100_diff1": 0.7709137968313824, + "nauc_map_at_100_max": 0.23417732525265697, + "nauc_map_at_100_std": -0.3529868072622312, + "nauc_map_at_10_diff1": 0.7735520846930982, + "nauc_map_at_10_max": 0.22722710833976106, + "nauc_map_at_10_std": -0.3707802906799404, + "nauc_map_at_1_diff1": 0.7944035377028345, + "nauc_map_at_1_max": 0.18453731579204632, + "nauc_map_at_1_std": -0.31459193294191756, + "nauc_map_at_20_diff1": 0.7720665962095318, + "nauc_map_at_20_max": 0.23121398453381503, + "nauc_map_at_20_std": -0.36048384547201145, + "nauc_map_at_3_diff1": 0.7773189801819204, + "nauc_map_at_3_max": 0.2089861737342013, + "nauc_map_at_3_std": -0.38697505834327794, + "nauc_map_at_5_diff1": 0.7738414220339891, + "nauc_map_at_5_max": 0.21745210492173755, + "nauc_map_at_5_std": -0.382621808062194, + "nauc_mrr_at_1000_diff1": 0.776983519127067, + "nauc_mrr_at_1000_max": 0.2494066473584712, + "nauc_mrr_at_1000_std": -0.31766166525611034, + "nauc_mrr_at_100_diff1": 0.776967736124463, + "nauc_mrr_at_100_max": 0.24939449119953552, + "nauc_mrr_at_100_std": -0.31766671314071193, + "nauc_mrr_at_10_diff1": 0.7769969707498353, + "nauc_mrr_at_10_max": 0.24973835082226808, + "nauc_mrr_at_10_std": -0.31926609297621594, + "nauc_mrr_at_1_diff1": 0.7828883043725049, + "nauc_mrr_at_1_max": 0.25016373889546284, + "nauc_mrr_at_1_std": -0.2940316688986738, + "nauc_mrr_at_20_diff1": 0.7770610400894614, + "nauc_mrr_at_20_max": 0.24971056592872423, + "nauc_mrr_at_20_std": -0.3178902173656083, + "nauc_mrr_at_3_diff1": 0.7762843372946363, + "nauc_mrr_at_3_max": 0.24926476114034066, + "nauc_mrr_at_3_std": -0.3240860325018437, + "nauc_mrr_at_5_diff1": 0.7766814059985857, + "nauc_mrr_at_5_max": 0.24835531623050705, + "nauc_mrr_at_5_std": -0.3216374930302086, + "nauc_ndcg_at_1000_diff1": 0.7696777944447858, + "nauc_ndcg_at_1000_max": 0.24218854847345309, + "nauc_ndcg_at_1000_std": -0.3374247891547267, + "nauc_ndcg_at_100_diff1": 0.7694826003526061, + "nauc_ndcg_at_100_max": 0.24059879650334898, + "nauc_ndcg_at_100_std": -0.33778035513403987, + "nauc_ndcg_at_10_diff1": 0.772280529279059, + "nauc_ndcg_at_10_max": 0.23864409616522123, + "nauc_ndcg_at_10_std": -0.3637251390369501, + "nauc_ndcg_at_1_diff1": 0.7832661106648008, + "nauc_ndcg_at_1_max": 0.25084766818780374, + "nauc_ndcg_at_1_std": -0.29468568226177405, + "nauc_ndcg_at_20_diff1": 0.7716551935289819, + "nauc_ndcg_at_20_max": 0.24070148451760634, + "nauc_ndcg_at_20_std": -0.35073264185669456, + "nauc_ndcg_at_3_diff1": 0.7693582386618356, + "nauc_ndcg_at_3_max": 0.22831832375706315, + "nauc_ndcg_at_3_std": -0.37193882195851025, + "nauc_ndcg_at_5_diff1": 0.770157761742659, + "nauc_ndcg_at_5_max": 0.2291173362852854, + "nauc_ndcg_at_5_std": -0.3729241926372365, + "nauc_precision_at_1000_diff1": -0.4412943949966939, + "nauc_precision_at_1000_max": 0.0036459404941215495, + "nauc_precision_at_1000_std": 0.31380531983454213, + "nauc_precision_at_100_diff1": -0.43811695386130445, + "nauc_precision_at_100_max": 0.00037949177387469374, + "nauc_precision_at_100_std": 0.3136431547347868, + "nauc_precision_at_10_diff1": -0.3868336391743595, + "nauc_precision_at_10_max": -0.0021210695423971426, + "nauc_precision_at_10_std": 0.2130974035423888, + "nauc_precision_at_1_diff1": 0.7832661106648008, + "nauc_precision_at_1_max": 0.25084766818780374, + "nauc_precision_at_1_std": -0.29468568226177405, + "nauc_precision_at_20_diff1": -0.41614751228443536, + "nauc_precision_at_20_max": -0.006430570950832867, + "nauc_precision_at_20_std": 0.2650727160002528, + "nauc_precision_at_3_diff1": -0.18499808457749403, + "nauc_precision_at_3_max": 0.04106593327180874, + "nauc_precision_at_3_std": 0.03720556044951809, + "nauc_precision_at_5_diff1": -0.3149822345207351, + "nauc_precision_at_5_max": 0.010136598528061673, + "nauc_precision_at_5_std": 0.1331511595759256, + "nauc_recall_at_1000_diff1": 0.4829244088575479, + "nauc_recall_at_1000_max": -0.060463626986179096, + "nauc_recall_at_1000_std": 0.15224554044983163, + "nauc_recall_at_100_diff1": 0.6528632210066015, + "nauc_recall_at_100_max": 0.07887343144626012, + "nauc_recall_at_100_std": -0.27506802657962964, + "nauc_recall_at_10_diff1": 0.745101806742001, + "nauc_recall_at_10_max": 0.19707577867857673, + "nauc_recall_at_10_std": -0.5348072417141793, + "nauc_recall_at_1_diff1": 0.7944035377028345, + "nauc_recall_at_1_max": 0.18453731579204632, + "nauc_recall_at_1_std": -0.31459193294191756, + "nauc_recall_at_20_diff1": 0.7496479880287762, + "nauc_recall_at_20_max": 0.2150439527585594, + "nauc_recall_at_20_std": -0.48242306432375026, + "nauc_recall_at_3_diff1": 0.745629712077306, + "nauc_recall_at_3_max": 0.17246905733933698, + "nauc_recall_at_3_std": -0.46444151430599806, + "nauc_recall_at_5_diff1": 0.7296056861757357, + "nauc_recall_at_5_max": 0.15986882659418775, + "nauc_recall_at_5_std": -0.5026880302584927, + "ndcg_at_1": 0.8205, + "ndcg_at_10": 0.89237, + "ndcg_at_100": 0.90316, + "ndcg_at_1000": 0.90395, + "ndcg_at_20": 0.89819, + "ndcg_at_3": 0.86449, + "ndcg_at_5": 0.88015, + "precision_at_1": 0.8205, + "precision_at_10": 0.13505, + "precision_at_100": 0.01538, + "precision_at_1000": 0.00157, + "precision_at_20": 0.07156, + "precision_at_3": 0.3773, + "precision_at_5": 0.24792, + "recall_at_1": 0.71553, + "recall_at_10": 0.96209, + "recall_at_100": 0.99659, + "recall_at_1000": 0.99992, + "recall_at_20": 0.97996, + "recall_at_3": 0.88315, + "recall_at_5": 0.92713 + } + ] + }, + "task_name": "QuoraRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RUParaPhraserSTS.json b/ai-sage__Giga-Embeddings-instruct/external/RUParaPhraserSTS.json new file mode 100644 index 000000000..c68e3e782 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 24.584528923034668, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.5990108783647319, + "cosine_spearman": 0.7047034713463798, + "euclidean_pearson": 0.6601030572916656, + "euclidean_spearman": 0.7047034713463798, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7047034713463798, + "manhattan_pearson": 0.6598001943092828, + "manhattan_spearman": 0.7042617367089709, + "pearson": 0.5990108783647319, + "spearman": 0.7047034713463798 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RiaNewsRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/RiaNewsRetrieval.json new file mode 100644 index 000000000..ff6476f22 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 9814.998483896255, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.71652, + "map_at_1": 0.5639, + "map_at_10": 0.6686, + "map_at_100": 0.67257, + "map_at_1000": 0.6727, + "map_at_20": 0.6712, + "map_at_3": 0.64823, + "map_at_5": 0.66074, + "mrr_at_1": 0.5639, + "mrr_at_10": 0.6686036507936463, + "mrr_at_100": 0.6725678485069567, + "mrr_at_1000": 0.6727020757906639, + "mrr_at_20": 0.6712014471381675, + "mrr_at_3": 0.6482333333333292, + "mrr_at_5": 0.6607433333333262, + "nauc_map_at_1000_diff1": 0.6174297820394619, + "nauc_map_at_1000_max": 0.09118854001349125, + "nauc_map_at_1000_std": -0.049435657872395504, + "nauc_map_at_100_diff1": 0.6173805378340941, + "nauc_map_at_100_max": 0.09112434713342751, + "nauc_map_at_100_std": -0.049296942464931826, + "nauc_map_at_10_diff1": 0.616542231505168, + "nauc_map_at_10_max": 0.09039711080932504, + "nauc_map_at_10_std": -0.05017750438974048, + "nauc_map_at_1_diff1": 0.6575155095156909, + "nauc_map_at_1_max": 0.09967117075442566, + "nauc_map_at_1_std": -0.06898771142001273, + "nauc_map_at_20_diff1": 0.6167226244130313, + "nauc_map_at_20_max": 0.09072225719428027, + "nauc_map_at_20_std": -0.04934926207117015, + "nauc_map_at_3_diff1": 0.6155367486454476, + "nauc_map_at_3_max": 0.09032982079978619, + "nauc_map_at_3_std": -0.0587535764632397, + "nauc_map_at_5_diff1": 0.6142683855708546, + "nauc_map_at_5_max": 0.08877070831850263, + "nauc_map_at_5_std": -0.054322768875290174, + "nauc_mrr_at_1000_diff1": 0.6174297820394619, + "nauc_mrr_at_1000_max": 0.09118854001349125, + "nauc_mrr_at_1000_std": -0.049435657872395504, + "nauc_mrr_at_100_diff1": 0.6173805378340941, + "nauc_mrr_at_100_max": 0.09112434713342751, + "nauc_mrr_at_100_std": -0.049296942464931826, + "nauc_mrr_at_10_diff1": 0.616542231505168, + "nauc_mrr_at_10_max": 0.09039711080932504, + "nauc_mrr_at_10_std": -0.05017750438974048, + "nauc_mrr_at_1_diff1": 0.6575155095156909, + "nauc_mrr_at_1_max": 0.09967117075442566, + "nauc_mrr_at_1_std": -0.06898771142001273, + "nauc_mrr_at_20_diff1": 0.6167226244130313, + "nauc_mrr_at_20_max": 0.09072225719428027, + "nauc_mrr_at_20_std": -0.04934926207117015, + "nauc_mrr_at_3_diff1": 0.6155367486454476, + "nauc_mrr_at_3_max": 0.09032982079978619, + "nauc_mrr_at_3_std": -0.0587535764632397, + "nauc_mrr_at_5_diff1": 0.6142683855708546, + "nauc_mrr_at_5_max": 0.08877070831850263, + "nauc_mrr_at_5_std": -0.054322768875290174, + "nauc_ndcg_at_1000_diff1": 0.6098630459378577, + "nauc_ndcg_at_1000_max": 0.09246035024166295, + "nauc_ndcg_at_1000_std": -0.03361612562569219, + "nauc_ndcg_at_100_diff1": 0.6084577656910963, + "nauc_ndcg_at_100_max": 0.09075671762808687, + "nauc_ndcg_at_100_std": -0.02862068237853277, + "nauc_ndcg_at_10_diff1": 0.6031150294618889, + "nauc_ndcg_at_10_max": 0.08705517607528171, + "nauc_ndcg_at_10_std": -0.03370467925663366, + "nauc_ndcg_at_1_diff1": 0.6575155095156909, + "nauc_ndcg_at_1_max": 0.09967117075442566, + "nauc_ndcg_at_1_std": -0.06898771142001273, + "nauc_ndcg_at_20_diff1": 0.6037124365840993, + "nauc_ndcg_at_20_max": 0.08823506745258437, + "nauc_ndcg_at_20_std": -0.02940751680095269, + "nauc_ndcg_at_3_diff1": 0.601413683201442, + "nauc_ndcg_at_3_max": 0.0868028808058021, + "nauc_ndcg_at_3_std": -0.054510327171744824, + "nauc_ndcg_at_5_diff1": 0.5979679000319666, + "nauc_ndcg_at_5_max": 0.08363528454727523, + "nauc_ndcg_at_5_std": -0.04548703878489744, + "nauc_precision_at_1000_diff1": 0.5071040464378512, + "nauc_precision_at_1000_max": 0.3039637541763784, + "nauc_precision_at_1000_std": 0.5440838986196943, + "nauc_precision_at_100_diff1": 0.5200580603436628, + "nauc_precision_at_100_max": 0.1099642529617639, + "nauc_precision_at_100_std": 0.3765801792134864, + "nauc_precision_at_10_diff1": 0.5284960081023135, + "nauc_precision_at_10_max": 0.06807738858421414, + "nauc_precision_at_10_std": 0.07489611728241942, + "nauc_precision_at_1_diff1": 0.6575155095156909, + "nauc_precision_at_1_max": 0.09967117075442566, + "nauc_precision_at_1_std": -0.06898771142001273, + "nauc_precision_at_20_diff1": 0.5102573126744686, + "nauc_precision_at_20_max": 0.07205985724189931, + "nauc_precision_at_20_std": 0.15162254397124517, + "nauc_precision_at_3_diff1": 0.5504812303914477, + "nauc_precision_at_3_max": 0.07391536062448881, + "nauc_precision_at_3_std": -0.038857451492220696, + "nauc_precision_at_5_diff1": 0.5274869245106691, + "nauc_precision_at_5_max": 0.060846073222739605, + "nauc_precision_at_5_std": -0.005062395770410055, + "nauc_recall_at_1000_diff1": 0.5071040464378544, + "nauc_recall_at_1000_max": 0.30396375417637206, + "nauc_recall_at_1000_std": 0.544083898619685, + "nauc_recall_at_100_diff1": 0.5200580603436653, + "nauc_recall_at_100_max": 0.10996425296176397, + "nauc_recall_at_100_std": 0.37658017921348175, + "nauc_recall_at_10_diff1": 0.5284960081023129, + "nauc_recall_at_10_max": 0.06807738858421314, + "nauc_recall_at_10_std": 0.07489611728242068, + "nauc_recall_at_1_diff1": 0.6575155095156909, + "nauc_recall_at_1_max": 0.09967117075442566, + "nauc_recall_at_1_std": -0.06898771142001273, + "nauc_recall_at_20_diff1": 0.5102573126744679, + "nauc_recall_at_20_max": 0.07205985724190103, + "nauc_recall_at_20_std": 0.15162254397124528, + "nauc_recall_at_3_diff1": 0.5504812303914475, + "nauc_recall_at_3_max": 0.0739153606244884, + "nauc_recall_at_3_std": -0.038857451492221015, + "nauc_recall_at_5_diff1": 0.5274869245106699, + "nauc_recall_at_5_max": 0.0608460732227402, + "nauc_recall_at_5_std": -0.0050623957704086005, + "ndcg_at_1": 0.5639, + "ndcg_at_10": 0.71652, + "ndcg_at_100": 0.73572, + "ndcg_at_1000": 0.73931, + "ndcg_at_20": 0.72598, + "ndcg_at_3": 0.67499, + "ndcg_at_5": 0.69755, + "precision_at_1": 0.5639, + "precision_at_10": 0.08655, + "precision_at_100": 0.00955, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04515, + "precision_at_3": 0.25073, + "precision_at_5": 0.1614, + "recall_at_1": 0.5639, + "recall_at_10": 0.8655, + "recall_at_100": 0.9551, + "recall_at_1000": 0.9834, + "recall_at_20": 0.9029, + "recall_at_3": 0.7522, + "recall_at_5": 0.807 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuBQReranking.json b/ai-sage__Giga-Embeddings-instruct/external/RuBQReranking.json new file mode 100644 index 000000000..849a977cd --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 516.0920803546906, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7666017430254295, + "map": 0.7666017430254295, + "mrr": 0.8153464738571121, + "nAUC_map_diff1": 0.3627891707625022, + "nAUC_map_max": 0.03203433802863856, + "nAUC_map_std": 0.06996438618963433, + "nAUC_mrr_diff1": 0.42679903988853113, + "nAUC_mrr_max": 0.07527480599102641, + "nAUC_mrr_std": 0.10097273643521257 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuBQRetrieval.json b/ai-sage__Giga-Embeddings-instruct/external/RuBQRetrieval.json new file mode 100644 index 000000000..b754cfa5e --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 352.1855094432831, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.73657, + "map_at_1": 0.44065, + "map_at_10": 0.65949, + "map_at_100": 0.66913, + "map_at_1000": 0.66929, + "map_at_20": 0.66636, + "map_at_3": 0.60377, + "map_at_5": 0.64036, + "mrr_at_1": 0.6247044917257684, + "mrr_at_10": 0.7410456302300272, + "mrr_at_100": 0.743642061621128, + "mrr_at_1000": 0.7436885014443636, + "mrr_at_20": 0.7427597096066316, + "mrr_at_3": 0.7228132387706853, + "mrr_at_5": 0.7351950354609932, + "nauc_map_at_1000_diff1": 0.37090200381007116, + "nauc_map_at_1000_max": 0.13910508855090156, + "nauc_map_at_1000_std": -0.18808239886229164, + "nauc_map_at_100_diff1": 0.3707829615924866, + "nauc_map_at_100_max": 0.13924480342518827, + "nauc_map_at_100_std": -0.18782458187728535, + "nauc_map_at_10_diff1": 0.3650603714813933, + "nauc_map_at_10_max": 0.13474364155052654, + "nauc_map_at_10_std": -0.19321769745629902, + "nauc_map_at_1_diff1": 0.4255523796946059, + "nauc_map_at_1_max": 0.08718694804676934, + "nauc_map_at_1_std": -0.1632083870750557, + "nauc_map_at_20_diff1": 0.36926962641332717, + "nauc_map_at_20_max": 0.13897116768015919, + "nauc_map_at_20_std": -0.18905045444291904, + "nauc_map_at_3_diff1": 0.3660441612374597, + "nauc_map_at_3_max": 0.11374638753689874, + "nauc_map_at_3_std": -0.18820705732888127, + "nauc_map_at_5_diff1": 0.3601240755195151, + "nauc_map_at_5_max": 0.12469431623670511, + "nauc_map_at_5_std": -0.19369051759609657, + "nauc_mrr_at_1000_diff1": 0.463294742344679, + "nauc_mrr_at_1000_max": 0.1733135504252722, + "nauc_mrr_at_1000_std": -0.22032497848622865, + "nauc_mrr_at_100_diff1": 0.463259637209302, + "nauc_mrr_at_100_max": 0.1734386391231157, + "nauc_mrr_at_100_std": -0.22024696828474613, + "nauc_mrr_at_10_diff1": 0.4616444815881303, + "nauc_mrr_at_10_max": 0.1727456729923913, + "nauc_mrr_at_10_std": -0.22185922618075202, + "nauc_mrr_at_1_diff1": 0.5041611103393129, + "nauc_mrr_at_1_max": 0.16019226770237172, + "nauc_mrr_at_1_std": -0.21554133105466397, + "nauc_mrr_at_20_diff1": 0.4626488921954804, + "nauc_mrr_at_20_max": 0.1735070812737816, + "nauc_mrr_at_20_std": -0.22055228499078322, + "nauc_mrr_at_3_diff1": 0.45916895100098337, + "nauc_mrr_at_3_max": 0.1733603270222595, + "nauc_mrr_at_3_std": -0.22138054343411756, + "nauc_mrr_at_5_diff1": 0.4595600551670136, + "nauc_mrr_at_5_max": 0.16876408776684326, + "nauc_mrr_at_5_std": -0.22553431986981592, + "nauc_ndcg_at_1000_diff1": 0.3890982879614141, + "nauc_ndcg_at_1000_max": 0.16062132022098236, + "nauc_ndcg_at_1000_std": -0.1882741879033837, + "nauc_ndcg_at_100_diff1": 0.3859230622276177, + "nauc_ndcg_at_100_max": 0.16479030069575218, + "nauc_ndcg_at_100_std": -0.1825862333167733, + "nauc_ndcg_at_10_diff1": 0.3664193020198755, + "nauc_ndcg_at_10_max": 0.1552146575883372, + "nauc_ndcg_at_10_std": -0.1992107820805643, + "nauc_ndcg_at_1_diff1": 0.505575101103112, + "nauc_ndcg_at_1_max": 0.16172381702653435, + "nauc_ndcg_at_1_std": -0.21334475676096917, + "nauc_ndcg_at_20_diff1": 0.3769618732698285, + "nauc_ndcg_at_20_max": 0.16546221585095636, + "nauc_ndcg_at_20_std": -0.18709641022125947, + "nauc_ndcg_at_3_diff1": 0.37126400354430666, + "nauc_ndcg_at_3_max": 0.12547246162551406, + "nauc_ndcg_at_3_std": -0.20396178391625852, + "nauc_ndcg_at_5_diff1": 0.35893501940516276, + "nauc_ndcg_at_5_max": 0.1349687833962685, + "nauc_ndcg_at_5_std": -0.20308812248088065, + "nauc_precision_at_1000_diff1": -0.0848188084168493, + "nauc_precision_at_1000_max": 0.06139161581867971, + "nauc_precision_at_1000_std": 0.040988421023932985, + "nauc_precision_at_100_diff1": -0.07885830971996956, + "nauc_precision_at_100_max": 0.08172209543497003, + "nauc_precision_at_100_std": 0.055370255660793115, + "nauc_precision_at_10_diff1": -0.03866974408944356, + "nauc_precision_at_10_max": 0.10976927790797955, + "nauc_precision_at_10_std": -0.019349418974770733, + "nauc_precision_at_1_diff1": 0.505575101103112, + "nauc_precision_at_1_max": 0.16172381702653435, + "nauc_precision_at_1_std": -0.21334475676096917, + "nauc_precision_at_20_diff1": -0.05637729717327838, + "nauc_precision_at_20_max": 0.10970959140054813, + "nauc_precision_at_20_std": 0.028388876862575005, + "nauc_precision_at_3_diff1": 0.09266489853955846, + "nauc_precision_at_3_max": 0.11106426174438222, + "nauc_precision_at_3_std": -0.1154429747011779, + "nauc_precision_at_5_diff1": 0.002889320715221165, + "nauc_precision_at_5_max": 0.1013303315463123, + "nauc_precision_at_5_std": -0.0670447500207753, + "nauc_recall_at_1000_diff1": 0.16221661299215462, + "nauc_recall_at_1000_max": 0.5359749132864255, + "nauc_recall_at_1000_std": 0.6464136682375486, + "nauc_recall_at_100_diff1": 0.12993211883064684, + "nauc_recall_at_100_max": 0.4555281725648927, + "nauc_recall_at_100_std": 0.2656347354237723, + "nauc_recall_at_10_diff1": 0.17779292640566433, + "nauc_recall_at_10_max": 0.1670396492397608, + "nauc_recall_at_10_std": -0.17078722406600041, + "nauc_recall_at_1_diff1": 0.4255523796946059, + "nauc_recall_at_1_max": 0.08718694804676934, + "nauc_recall_at_1_std": -0.1632083870750557, + "nauc_recall_at_20_diff1": 0.17120065267465978, + "nauc_recall_at_20_max": 0.2518437803181794, + "nauc_recall_at_20_std": -0.0782129157238653, + "nauc_recall_at_3_diff1": 0.2651229427509311, + "nauc_recall_at_3_max": 0.0947296410240245, + "nauc_recall_at_3_std": -0.17912490817170942, + "nauc_recall_at_5_diff1": 0.21037615823181474, + "nauc_recall_at_5_max": 0.10398109095863245, + "nauc_recall_at_5_std": -0.181775717992346, + "ndcg_at_1": 0.62411, + "ndcg_at_10": 0.73657, + "ndcg_at_100": 0.7645, + "ndcg_at_1000": 0.76724, + "ndcg_at_20": 0.75241, + "ndcg_at_3": 0.66531, + "ndcg_at_5": 0.70545, + "precision_at_1": 0.62411, + "precision_at_10": 0.14309, + "precision_at_100": 0.0164, + "precision_at_1000": 0.00167, + "precision_at_20": 0.07686, + "precision_at_3": 0.36131, + "precision_at_5": 0.25414, + "recall_at_1": 0.44065, + "recall_at_10": 0.87569, + "recall_at_100": 0.97866, + "recall_at_1000": 0.99663, + "recall_at_20": 0.92614, + "recall_at_3": 0.70376, + "recall_at_5": 0.79596 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuReviewsClassification.json b/ai-sage__Giga-Embeddings-instruct/external/RuReviewsClassification.json new file mode 100644 index 000000000..3745e99fc --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuReviewsClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", "evaluation_time": 99.49517941474915, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.737150390625, "f1": 0.722246927657427, "f1_weighted": 0.7222538385584938, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.737150390625, "scores_per_experiment": [{"accuracy": 0.76727734375, "f1": 0.7556766444627057, "f1_weighted": 0.7556801472217148}, {"accuracy": 0.7335859375, "f1": 0.7208908137982117, "f1_weighted": 0.7208965683166522}, {"accuracy": 0.70770703125, "f1": 0.6920992346572302, "f1_weighted": 0.6921133052933363}, {"accuracy": 0.75751171875, "f1": 0.7502995745063169, "f1_weighted": 0.7503294558590532}, {"accuracy": 0.76337109375, "f1": 0.7562326267137918, "f1_weighted": 0.7562530032353445}, {"accuracy": 0.73114453125, "f1": 0.7076274920705875, "f1_weighted": 0.7076177281733378}, {"accuracy": 0.72528515625, "f1": 0.7018444881807873, "f1_weighted": 0.7018417559480226}, {"accuracy": 0.72626171875, "f1": 0.7139465882627896, "f1_weighted": 0.7139673545361157}, {"accuracy": 0.732609375, "f1": 0.7106761864811024, "f1_weighted": 0.7106545647425278}, {"accuracy": 0.72675, "f1": 0.7131756274407471, "f1_weighted": 0.7131845022588335}]}]}, "task_name": "RuReviewsClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuSTSBenchmarkSTS.json b/ai-sage__Giga-Embeddings-instruct/external/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..b4a9d3a2d --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 22.67481565475464, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8099594416064292, + "cosine_spearman": 0.7930076871318473, + "euclidean_pearson": 0.8007768395803017, + "euclidean_spearman": 0.7930076871318473, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7930076871318473, + "manhattan_pearson": 0.8014948537837334, + "manhattan_spearman": 0.7937609905744646, + "pearson": 0.8099594416064292, + "spearman": 0.7930076871318473 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClassification.json b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..3c4bf4aaf --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", "evaluation_time": 188.23047757148743, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.74520703125, "f1": 0.7307891907358736, "f1_weighted": 0.7309099552030747, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.74520703125, "scores_per_experiment": [{"accuracy": 0.74628125, "f1": 0.7318016173228531, "f1_weighted": 0.7319055576392282}, {"accuracy": 0.74969921875, "f1": 0.7351409618361473, "f1_weighted": 0.7352686554276316}, {"accuracy": 0.73114453125, "f1": 0.7150292142483131, "f1_weighted": 0.7151869375282435}, {"accuracy": 0.7492109375, "f1": 0.7349738301952348, "f1_weighted": 0.735087481916472}, {"accuracy": 0.75067578125, "f1": 0.7367660440367951, "f1_weighted": 0.7368826649904742}, {"accuracy": 0.74383984375, "f1": 0.7302542631283219, "f1_weighted": 0.7303628364685226}, {"accuracy": 0.74774609375, "f1": 0.7306573041651887, "f1_weighted": 0.7307978893704438}, {"accuracy": 0.74383984375, "f1": 0.7273070551857849, "f1_weighted": 0.7274518873927309}, {"accuracy": 0.752140625, "f1": 0.7413547916751491, "f1_weighted": 0.7414490489933776}, {"accuracy": 0.7374921875, "f1": 0.7246068255649476, "f1_weighted": 0.724706592303623}]}]}, "task_name": "RuSciBenchGRNTIClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClusteringP2P.json b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..1ab10d977 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 94.75362753868103, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.673025150400559, + "v_measure": 0.673025150400559, + "v_measure_std": 0.010050822060573549, + "v_measures": { + "Level 0": [ + 0.6709706162528627, + 0.6879580427776311, + 0.6749176768037941, + 0.6551029518231402, + 0.6710753419811606, + 0.6679579556848002, + 0.6852927918576992, + 0.6858334123956298, + 0.6668058843405836, + 0.6643368300882878 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClassification.json b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..7afe600c5 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClassification.json @@ -0,0 +1 @@ +{"dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", "evaluation_time": 188.13478350639343, "kg_co2_emissions": null, "mteb_version": "1.15.3", "scores": {"test": [{"accuracy": 0.61512890625, "f1": 0.5940504694034752, "f1_weighted": 0.5940999969456683, "hf_subset": "default", "languages": ["rus-Cyrl"], "main_score": 0.61512890625, "scores_per_experiment": [{"accuracy": 0.6203046875, "f1": 0.5951240996812499, "f1_weighted": 0.5952955327984815}, {"accuracy": 0.61200390625, "f1": 0.5916008303025464, "f1_weighted": 0.5916763175620705}, {"accuracy": 0.62128125, "f1": 0.6016488845595703, "f1_weighted": 0.601683674906823}, {"accuracy": 0.59540234375, "f1": 0.562730209274492, "f1_weighted": 0.5626927062089637}, {"accuracy": 0.6163984375, "f1": 0.5954513758581562, "f1_weighted": 0.5955162839605828}, {"accuracy": 0.6251875, "f1": 0.6065551503319685, "f1_weighted": 0.6066194866662501}, {"accuracy": 0.61786328125, "f1": 0.5992076313912165, "f1_weighted": 0.5993248030410878}, {"accuracy": 0.623234375, "f1": 0.6061181617297106, "f1_weighted": 0.6061372829949966}, {"accuracy": 0.6007734375, "f1": 0.5792843538552442, "f1_weighted": 0.5792814380518859}, {"accuracy": 0.61883984375, "f1": 0.6027839970505974, "f1_weighted": 0.6027724432655408}]}]}, "task_name": "RuSciBenchOECDClassification"} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClusteringP2P.json b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..533566538 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 92.69741201400757, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.566947461499336, + "v_measure": 0.566947461499336, + "v_measure_std": 0.005136380257006159, + "v_measures": { + "Level 0": [ + 0.5714296040671906, + 0.5611783641579191, + 0.5747375884820992, + 0.5709788584298044, + 0.5636100229915219, + 0.5666355496660465, + 0.5576319612909203, + 0.5637831729173005, + 0.5675269730794281, + 0.5719625199111297 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SCIDOCS.json b/ai-sage__Giga-Embeddings-instruct/external/SCIDOCS.json new file mode 100644 index 000000000..1c142c74c --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "evaluation_time": 319.0548052787781, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.19162, + "map_at_1": 0.04475, + "map_at_10": 0.11306, + "map_at_100": 0.13628, + "map_at_1000": 0.13977, + "map_at_20": 0.1241, + "map_at_3": 0.07941, + "map_at_5": 0.09681, + "mrr_at_1": 0.222, + "mrr_at_10": 0.32707182539682517, + "mrr_at_100": 0.33944931964999925, + "mrr_at_1000": 0.3399928862072715, + "mrr_at_20": 0.3342188715996763, + "mrr_at_3": 0.2903333333333334, + "mrr_at_5": 0.313883333333333, + "nauc_map_at_1000_diff1": 0.16360578088282296, + "nauc_map_at_1000_max": 0.24255546647291373, + "nauc_map_at_1000_std": 0.23653561658845026, + "nauc_map_at_100_diff1": 0.16194025097940373, + "nauc_map_at_100_max": 0.24085864603626417, + "nauc_map_at_100_std": 0.23364987990124178, + "nauc_map_at_10_diff1": 0.17166987284706134, + "nauc_map_at_10_max": 0.239828055787183, + "nauc_map_at_10_std": 0.2094729363065384, + "nauc_map_at_1_diff1": 0.26524260461587695, + "nauc_map_at_1_max": 0.18237790264396608, + "nauc_map_at_1_std": 0.13862573541948847, + "nauc_map_at_20_diff1": 0.16714123094555536, + "nauc_map_at_20_max": 0.23718224495718734, + "nauc_map_at_20_std": 0.21870044423697838, + "nauc_map_at_3_diff1": 0.19512270329933146, + "nauc_map_at_3_max": 0.20855620393884433, + "nauc_map_at_3_std": 0.16035011072648478, + "nauc_map_at_5_diff1": 0.1882304971891875, + "nauc_map_at_5_max": 0.21734182123122006, + "nauc_map_at_5_std": 0.18044239578716523, + "nauc_mrr_at_1000_diff1": 0.2080051056261849, + "nauc_mrr_at_1000_max": 0.18295945240932288, + "nauc_mrr_at_1000_std": 0.1728025318943188, + "nauc_mrr_at_100_diff1": 0.20790178515273827, + "nauc_mrr_at_100_max": 0.1831986327791731, + "nauc_mrr_at_100_std": 0.17320091305114396, + "nauc_mrr_at_10_diff1": 0.20852629583546498, + "nauc_mrr_at_10_max": 0.18196816934222146, + "nauc_mrr_at_10_std": 0.17068040995717965, + "nauc_mrr_at_1_diff1": 0.2644027283676547, + "nauc_mrr_at_1_max": 0.1804942766822405, + "nauc_mrr_at_1_std": 0.13892939744935398, + "nauc_mrr_at_20_diff1": 0.2074305901889222, + "nauc_mrr_at_20_max": 0.18201319332283358, + "nauc_mrr_at_20_std": 0.17323956767473866, + "nauc_mrr_at_3_diff1": 0.2080860026418777, + "nauc_mrr_at_3_max": 0.1793980894932549, + "nauc_mrr_at_3_std": 0.16018953507592426, + "nauc_mrr_at_5_diff1": 0.20664338981633468, + "nauc_mrr_at_5_max": 0.1799219934317394, + "nauc_mrr_at_5_std": 0.16872977356765642, + "nauc_ndcg_at_1000_diff1": 0.15863416705505543, + "nauc_ndcg_at_1000_max": 0.24110686464543268, + "nauc_ndcg_at_1000_std": 0.2828638556655814, + "nauc_ndcg_at_100_diff1": 0.14680972591439675, + "nauc_ndcg_at_100_max": 0.23239600232159474, + "nauc_ndcg_at_100_std": 0.27111251722669194, + "nauc_ndcg_at_10_diff1": 0.16420888690623528, + "nauc_ndcg_at_10_max": 0.22751709784752375, + "nauc_ndcg_at_10_std": 0.21453901377928283, + "nauc_ndcg_at_1_diff1": 0.2644027283676547, + "nauc_ndcg_at_1_max": 0.1804942766822405, + "nauc_ndcg_at_1_std": 0.13892939744935398, + "nauc_ndcg_at_20_diff1": 0.15811379241678267, + "nauc_ndcg_at_20_max": 0.22240678546737125, + "nauc_ndcg_at_20_std": 0.23198241749302495, + "nauc_ndcg_at_3_diff1": 0.18504832287432113, + "nauc_ndcg_at_3_max": 0.1975223586077359, + "nauc_ndcg_at_3_std": 0.16473921583621845, + "nauc_ndcg_at_5_diff1": 0.17904056465395726, + "nauc_ndcg_at_5_max": 0.20684461097896245, + "nauc_ndcg_at_5_std": 0.1858866622503915, + "nauc_precision_at_1000_diff1": 0.06193891363199702, + "nauc_precision_at_1000_max": 0.2085287311151861, + "nauc_precision_at_1000_std": 0.3753891581521263, + "nauc_precision_at_100_diff1": 0.0597130047912924, + "nauc_precision_at_100_max": 0.20806413594542678, + "nauc_precision_at_100_std": 0.3340002816119094, + "nauc_precision_at_10_diff1": 0.11257140852641279, + "nauc_precision_at_10_max": 0.2340935798614799, + "nauc_precision_at_10_std": 0.24021792174750078, + "nauc_precision_at_1_diff1": 0.2644027283676547, + "nauc_precision_at_1_max": 0.1804942766822405, + "nauc_precision_at_1_std": 0.13892939744935398, + "nauc_precision_at_20_diff1": 0.09777581420498402, + "nauc_precision_at_20_max": 0.2094783417188155, + "nauc_precision_at_20_std": 0.26155229221480036, + "nauc_precision_at_3_diff1": 0.14815195778747578, + "nauc_precision_at_3_max": 0.20297896590597084, + "nauc_precision_at_3_std": 0.1727297537409199, + "nauc_precision_at_5_diff1": 0.13929822537338526, + "nauc_precision_at_5_max": 0.20988014452928389, + "nauc_precision_at_5_std": 0.2036762932119487, + "nauc_recall_at_1000_diff1": 0.06269540116426191, + "nauc_recall_at_1000_max": 0.2250245807435009, + "nauc_recall_at_1000_std": 0.3892843130214358, + "nauc_recall_at_100_diff1": 0.058585623241962134, + "nauc_recall_at_100_max": 0.21146719698636726, + "nauc_recall_at_100_std": 0.3340917884578359, + "nauc_recall_at_10_diff1": 0.10791079221016839, + "nauc_recall_at_10_max": 0.23402958626226222, + "nauc_recall_at_10_std": 0.23778905836972059, + "nauc_recall_at_1_diff1": 0.26524260461587695, + "nauc_recall_at_1_max": 0.18237790264396608, + "nauc_recall_at_1_std": 0.13862573541948847, + "nauc_recall_at_20_diff1": 0.09588806834662644, + "nauc_recall_at_20_max": 0.20992020581818324, + "nauc_recall_at_20_std": 0.2592296591496667, + "nauc_recall_at_3_diff1": 0.1486565067742853, + "nauc_recall_at_3_max": 0.20312249816233308, + "nauc_recall_at_3_std": 0.16924697629563207, + "nauc_recall_at_5_diff1": 0.13735197736933963, + "nauc_recall_at_5_max": 0.2094698008166532, + "nauc_recall_at_5_std": 0.19962609439107787, + "ndcg_at_1": 0.222, + "ndcg_at_10": 0.19162, + "ndcg_at_100": 0.28321, + "ndcg_at_1000": 0.34267, + "ndcg_at_20": 0.22254, + "ndcg_at_3": 0.17798, + "ndcg_at_5": 0.15975, + "precision_at_1": 0.222, + "precision_at_10": 0.0998, + "precision_at_100": 0.02329, + "precision_at_1000": 0.00376, + "precision_at_20": 0.06795, + "precision_at_3": 0.16567, + "precision_at_5": 0.1424, + "recall_at_1": 0.04475, + "recall_at_10": 0.20307, + "recall_at_100": 0.47242, + "recall_at_1000": 0.76242, + "recall_at_20": 0.27597, + "recall_at_3": 0.1005, + "recall_at_5": 0.14432 + } + ] + }, + "task_name": "SCIDOCS" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SICK-R.json b/ai-sage__Giga-Embeddings-instruct/external/SICK-R.json new file mode 100644 index 000000000..6ae01def7 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "evaluation_time": 44.206942558288574, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8557114189789801, + "cosine_spearman": 0.8106253068558962, + "euclidean_pearson": 0.8327075641353736, + "euclidean_spearman": 0.8106253622684206, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8106253068558962, + "manhattan_pearson": 0.8335659901147551, + "manhattan_spearman": 0.8113591158643306, + "pearson": 0.8557114189789801, + "spearman": 0.8106253068558962 + } + ] + }, + "task_name": "SICK-R" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS12.json b/ai-sage__Giga-Embeddings-instruct/external/STS12.json new file mode 100644 index 000000000..59bfdc2a7 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "evaluation_time": 26.36725664138794, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8178267795485767, + "cosine_spearman": 0.7009877188862462, + "euclidean_pearson": 0.7849440835422977, + "euclidean_spearman": 0.7009877188862462, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7009877188862462, + "manhattan_pearson": 0.787282306108082, + "manhattan_spearman": 0.7029557277377175, + "pearson": 0.8178267795485767, + "spearman": 0.7009877188862462 + } + ] + }, + "task_name": "STS12" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS13.json b/ai-sage__Giga-Embeddings-instruct/external/STS13.json new file mode 100644 index 000000000..a2e42f939 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "evaluation_time": 21.206116437911987, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8025745663712165, + "cosine_spearman": 0.8101483036919428, + "euclidean_pearson": 0.8114541657219165, + "euclidean_spearman": 0.8101478607738545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8101483036919428, + "manhattan_pearson": 0.8140111412290987, + "manhattan_spearman": 0.8128649812590226, + "pearson": 0.8025745663712165, + "spearman": 0.8101483036919428 + } + ] + }, + "task_name": "STS13" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS14.json b/ai-sage__Giga-Embeddings-instruct/external/STS14.json new file mode 100644 index 000000000..915b6f7ea --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "evaluation_time": 27.161380767822266, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.817451052491344, + "cosine_spearman": 0.7781666029040504, + "euclidean_pearson": 0.8131747977745774, + "euclidean_spearman": 0.7781666029040504, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7781666029040504, + "manhattan_pearson": 0.8146161935201552, + "manhattan_spearman": 0.7798131880904402, + "pearson": 0.817451052491344, + "spearman": 0.7781666029040504 + } + ] + }, + "task_name": "STS14" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS15.json b/ai-sage__Giga-Embeddings-instruct/external/STS15.json new file mode 100644 index 000000000..a18b9d66c --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "evaluation_time": 26.17412495613098, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8369084648758516, + "cosine_spearman": 0.8452134445704115, + "euclidean_pearson": 0.8413552443709718, + "euclidean_spearman": 0.8452134445704115, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8452134445704115, + "manhattan_pearson": 0.842196680605194, + "manhattan_spearman": 0.8461306373914835, + "pearson": 0.8369084648758516, + "spearman": 0.8452134445704115 + } + ] + }, + "task_name": "STS15" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS16.json b/ai-sage__Giga-Embeddings-instruct/external/STS16.json new file mode 100644 index 000000000..0b0b9951d --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS16.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513", + "evaluation_time": 21.583980798721313, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.7922411263296355, + "cosine_spearman": 0.810920748151385, + "euclidean_pearson": 0.8069903573722832, + "euclidean_spearman": 0.810920748151385, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.810920748151385, + "manhattan_pearson": 0.8082176565456413, + "manhattan_spearman": 0.8122812310589154, + "pearson": 0.7922411263296355, + "spearman": 0.810920748151385 + } + ] + }, + "task_name": "STS16" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STS22.json b/ai-sage__Giga-Embeddings-instruct/external/STS22.json new file mode 100644 index 000000000..6d7801e3d --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STS22.json @@ -0,0 +1,90 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 229.31791186332703, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.5565638854673621, + "cosine_spearman": 0.678978765123834, + "euclidean_pearson": 0.60912723499174, + "euclidean_spearman": 0.678978765123834, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.678978765123834, + "manhattan_pearson": 0.6243178407548512, + "manhattan_spearman": 0.6828262888978559, + "pearson": 0.5565638854673621, + "spearman": 0.678978765123834 + }, + { + "cosine_pearson": 0.6386172304063621, + "cosine_spearman": 0.6415805209466257, + "euclidean_pearson": 0.6474688851141593, + "euclidean_spearman": 0.6415805209466257, + "hf_subset": "pl-en", + "languages": [ + "pol-Latn", + "eng-Latn" + ], + "main_score": 0.6415805209466257, + "manhattan_pearson": 0.6612890961014374, + "manhattan_spearman": 0.6479879912353912, + "pearson": 0.6386172304063621, + "spearman": 0.6415805209466257 + }, + { + "cosine_pearson": 0.6323427935966728, + "cosine_spearman": 0.6362411689657603, + "euclidean_pearson": 0.6591000068718458, + "euclidean_spearman": 0.6362411689657603, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6362411689657603, + "manhattan_pearson": 0.6610293101409169, + "manhattan_spearman": 0.6378745954139023, + "pearson": 0.6323427935966728, + "spearman": 0.6362411689657603 + }, + { + "cosine_pearson": 0.609260222043179, + "cosine_spearman": 0.6475327857364532, + "euclidean_pearson": 0.6269516497034605, + "euclidean_spearman": 0.6475327857364532, + "hf_subset": "zh-en", + "languages": [ + "cmn-Hans", + "eng-Latn" + ], + "main_score": 0.6475327857364532, + "manhattan_pearson": 0.6390894015427736, + "manhattan_spearman": 0.6548329498299246, + "pearson": 0.609260222043179, + "spearman": 0.6475327857364532 + }, + { + "cosine_pearson": 0.5269990619148788, + "cosine_spearman": 0.5433450120165593, + "euclidean_pearson": 0.5832806987651424, + "euclidean_spearman": 0.5433450120165593, + "hf_subset": "de-en", + "languages": [ + "deu-Latn", + "eng-Latn" + ], + "main_score": 0.5433450120165593, + "manhattan_pearson": 0.6000420951688084, + "manhattan_spearman": 0.549659883866187, + "pearson": 0.5269990619148788, + "spearman": 0.5433450120165593 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/STSBenchmark.json b/ai-sage__Giga-Embeddings-instruct/external/STSBenchmark.json new file mode 100644 index 000000000..f67bf0597 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "evaluation_time": 21.454996347427368, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.8225427804989813, + "cosine_spearman": 0.8220135109121626, + "euclidean_pearson": 0.8321943775385887, + "euclidean_spearman": 0.822013271737365, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8220135109121626, + "manhattan_pearson": 0.8330139032750423, + "manhattan_spearman": 0.82313788704099, + "pearson": 0.8225427804989813, + "spearman": 0.8220135109121626 + } + ] + }, + "task_name": "STSBenchmark" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SciDocsRR.json b/ai-sage__Giga-Embeddings-instruct/external/SciDocsRR.json new file mode 100644 index 000000000..11cefdbf6 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SciDocsRR.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", + "evaluation_time": 177.69396018981934, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8800790793717359, + "map": 0.8800790793717359, + "mrr": 0.962317149081855, + "nAUC_map_diff1": -0.04285706894005613, + "nAUC_map_max": 0.4769800035323553, + "nAUC_map_std": 0.6605509605566249, + "nAUC_mrr_diff1": 0.37593272739783384, + "nAUC_mrr_max": 0.7483336690604901, + "nAUC_mrr_std": 0.7976400854713901 + } + ] + }, + "task_name": "SciDocsRR" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SciFact.json b/ai-sage__Giga-Embeddings-instruct/external/SciFact.json new file mode 100644 index 000000000..692f1cc44 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SciFact.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "evaluation_time": 97.03665351867676, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.72902, + "map_at_1": 0.58161, + "map_at_10": 0.6815, + "map_at_100": 0.68763, + "map_at_1000": 0.68777, + "map_at_20": 0.68636, + "map_at_3": 0.65496, + "map_at_5": 0.67166, + "mrr_at_1": 0.6166666666666667, + "mrr_at_10": 0.6990119047619047, + "mrr_at_100": 0.7035924850016259, + "mrr_at_1000": 0.7036956277343268, + "mrr_at_20": 0.702427525490219, + "mrr_at_3": 0.68, + "mrr_at_5": 0.6923333333333332, + "nauc_map_at_1000_diff1": 0.6760995698057088, + "nauc_map_at_1000_max": 0.46068809020612705, + "nauc_map_at_1000_std": 0.011948672845462098, + "nauc_map_at_100_diff1": 0.6760952762159904, + "nauc_map_at_100_max": 0.4607379744229154, + "nauc_map_at_100_std": 0.01182970323131957, + "nauc_map_at_10_diff1": 0.6757824321545666, + "nauc_map_at_10_max": 0.46326127505244935, + "nauc_map_at_10_std": 0.012781320788709092, + "nauc_map_at_1_diff1": 0.705130136513353, + "nauc_map_at_1_max": 0.4067340888833154, + "nauc_map_at_1_std": -0.013408323300249747, + "nauc_map_at_20_diff1": 0.6752032735442157, + "nauc_map_at_20_max": 0.4612651713618736, + "nauc_map_at_20_std": 0.011355480686211972, + "nauc_map_at_3_diff1": 0.6816380401477546, + "nauc_map_at_3_max": 0.43886805631848513, + "nauc_map_at_3_std": -0.03720201195392404, + "nauc_map_at_5_diff1": 0.6799170514346827, + "nauc_map_at_5_max": 0.451182341963952, + "nauc_map_at_5_std": -0.008736684979875235, + "nauc_mrr_at_1000_diff1": 0.6559068609451051, + "nauc_mrr_at_1000_max": 0.4756601805618958, + "nauc_mrr_at_1000_std": 0.05913689253819713, + "nauc_mrr_at_100_diff1": 0.6559130506472648, + "nauc_mrr_at_100_max": 0.47560766539048227, + "nauc_mrr_at_100_std": 0.05886452280327445, + "nauc_mrr_at_10_diff1": 0.6538789898290563, + "nauc_mrr_at_10_max": 0.4782853216971601, + "nauc_mrr_at_10_std": 0.06362470864688538, + "nauc_mrr_at_1_diff1": 0.6764455543946382, + "nauc_mrr_at_1_max": 0.44873474172421685, + "nauc_mrr_at_1_std": 0.07448841772635736, + "nauc_mrr_at_20_diff1": 0.6547605317912657, + "nauc_mrr_at_20_max": 0.47587096680661467, + "nauc_mrr_at_20_std": 0.05809216165052655, + "nauc_mrr_at_3_diff1": 0.6577227321919353, + "nauc_mrr_at_3_max": 0.4754231580750389, + "nauc_mrr_at_3_std": 0.04222335116280824, + "nauc_mrr_at_5_diff1": 0.6554309281879533, + "nauc_mrr_at_5_max": 0.4789198835887271, + "nauc_mrr_at_5_std": 0.05990782526521852, + "nauc_ndcg_at_1000_diff1": 0.6650501237344043, + "nauc_ndcg_at_1000_max": 0.4789691614927347, + "nauc_ndcg_at_1000_std": 0.03858567312387061, + "nauc_ndcg_at_100_diff1": 0.6652978786688968, + "nauc_ndcg_at_100_max": 0.4792798566987145, + "nauc_ndcg_at_100_std": 0.03282731643146015, + "nauc_ndcg_at_10_diff1": 0.6558633101511163, + "nauc_ndcg_at_10_max": 0.48802885201057816, + "nauc_ndcg_at_10_std": 0.039764819883604384, + "nauc_ndcg_at_1_diff1": 0.6764455543946382, + "nauc_ndcg_at_1_max": 0.44873474172421685, + "nauc_ndcg_at_1_std": 0.07448841772635736, + "nauc_ndcg_at_20_diff1": 0.6567440742057199, + "nauc_ndcg_at_20_max": 0.4811394986776404, + "nauc_ndcg_at_20_std": 0.02711393534038602, + "nauc_ndcg_at_3_diff1": 0.665498828618974, + "nauc_ndcg_at_3_max": 0.45788427202371434, + "nauc_ndcg_at_3_std": -0.025245039228722913, + "nauc_ndcg_at_5_diff1": 0.6651681687231908, + "nauc_ndcg_at_5_max": 0.47282390021042286, + "nauc_ndcg_at_5_std": 0.008797704241318283, + "nauc_precision_at_1000_diff1": -0.25278551984459546, + "nauc_precision_at_1000_max": 0.19072121412932663, + "nauc_precision_at_1000_std": 0.510326726047779, + "nauc_precision_at_100_diff1": -0.17955055628472952, + "nauc_precision_at_100_max": 0.22786207573322784, + "nauc_precision_at_100_std": 0.4552886915349048, + "nauc_precision_at_10_diff1": 0.07786315249522727, + "nauc_precision_at_10_max": 0.4045721598510848, + "nauc_precision_at_10_std": 0.38868118095620857, + "nauc_precision_at_1_diff1": 0.6764455543946382, + "nauc_precision_at_1_max": 0.44873474172421685, + "nauc_precision_at_1_std": 0.07448841772635736, + "nauc_precision_at_20_diff1": -0.06778747709882627, + "nauc_precision_at_20_max": 0.2943636167690474, + "nauc_precision_at_20_std": 0.3848151740489274, + "nauc_precision_at_3_diff1": 0.39481963258187014, + "nauc_precision_at_3_max": 0.4242369906705573, + "nauc_precision_at_3_std": 0.09531028412147295, + "nauc_precision_at_5_diff1": 0.24898111796597416, + "nauc_precision_at_5_max": 0.4117351707929381, + "nauc_precision_at_5_std": 0.2490291910370421, + "nauc_recall_at_1000_diff1": 0.7076330532212898, + "nauc_recall_at_1000_max": 0.8885387488328592, + "nauc_recall_at_1000_std": 0.8325163398692724, + "nauc_recall_at_100_diff1": 0.6808723489395772, + "nauc_recall_at_100_max": 0.6374994442221321, + "nauc_recall_at_100_std": 0.07758659019163229, + "nauc_recall_at_10_diff1": 0.5654551862159732, + "nauc_recall_at_10_max": 0.5674870654004741, + "nauc_recall_at_10_std": 0.08701951811447595, + "nauc_recall_at_1_diff1": 0.705130136513353, + "nauc_recall_at_1_max": 0.4067340888833154, + "nauc_recall_at_1_std": -0.013408323300249747, + "nauc_recall_at_20_diff1": 0.5413998932906485, + "nauc_recall_at_20_max": 0.5545440398381579, + "nauc_recall_at_20_std": -0.0317052747024727, + "nauc_recall_at_3_diff1": 0.6326092590929955, + "nauc_recall_at_3_max": 0.46055085657367106, + "nauc_recall_at_3_std": -0.10957671894931113, + "nauc_recall_at_5_diff1": 0.6162947530846916, + "nauc_recall_at_5_max": 0.5048704479871539, + "nauc_recall_at_5_std": -0.014810663364085042, + "ndcg_at_1": 0.61667, + "ndcg_at_10": 0.72902, + "ndcg_at_100": 0.75368, + "ndcg_at_1000": 0.75657, + "ndcg_at_20": 0.74453, + "ndcg_at_3": 0.6857, + "ndcg_at_5": 0.70926, + "precision_at_1": 0.61667, + "precision_at_10": 0.09667, + "precision_at_100": 0.01093, + "precision_at_1000": 0.00112, + "precision_at_20": 0.052, + "precision_at_3": 0.27, + "precision_at_5": 0.178, + "recall_at_1": 0.58161, + "recall_at_10": 0.85689, + "recall_at_100": 0.965, + "recall_at_1000": 0.98667, + "recall_at_20": 0.916, + "recall_at_3": 0.74083, + "recall_at_5": 0.80022 + } + ] + }, + "task_name": "SciFact" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SensitiveTopicsClassification.json b/ai-sage__Giga-Embeddings-instruct/external/SensitiveTopicsClassification.json new file mode 100644 index 000000000..bb2a9aec7 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 42.21324563026428, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.30009765625, + "f1": 0.3197592344768223, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.46344197591145064, + "main_score": 0.30009765625, + "scores_per_experiment": [ + { + "accuracy": 0.287109375, + "f1": 0.27384453110940715, + "lrap": 0.4298638237847143 + }, + { + "accuracy": 0.3037109375, + "f1": 0.360910078774308, + "lrap": 0.4832899305555484 + }, + { + "accuracy": 0.30908203125, + "f1": 0.2905950362836898, + "lrap": 0.44978841145832527 + }, + { + "accuracy": 0.2978515625, + "f1": 0.3319560381408308, + "lrap": 0.4584282769097145 + }, + { + "accuracy": 0.287109375, + "f1": 0.3231070154642379, + "lrap": 0.4499172634548535 + }, + { + "accuracy": 0.31201171875, + "f1": 0.3170050397429188, + "lrap": 0.48608398437499273 + }, + { + "accuracy": 0.31005859375, + "f1": 0.32177193732294695, + "lrap": 0.4827541775173535 + }, + { + "accuracy": 0.28515625, + "f1": 0.32428650959500377, + "lrap": 0.4545423719617979 + }, + { + "accuracy": 0.3095703125, + "f1": 0.3286180138000024, + "lrap": 0.4683837890624923 + }, + { + "accuracy": 0.29931640625, + "f1": 0.3254981445348768, + "lrap": 0.4713677300347145 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SprintDuplicateQuestions.json b/ai-sage__Giga-Embeddings-instruct/external/SprintDuplicateQuestions.json new file mode 100644 index 000000000..f44f40fad --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "evaluation_time": 26.336684942245483, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_accuracy": 0.9960594059405941, + "cosine_accuracy_threshold": 0.8066068887710571, + "cosine_ap": 0.8629950883344696, + "cosine_f1": 0.7928020565552699, + "cosine_f1_threshold": 0.8031163215637207, + "cosine_precision": 0.8158730158730159, + "cosine_recall": 0.771, + "dot_accuracy": 0.9960594059405941, + "dot_accuracy_threshold": 0.8066068291664124, + "dot_ap": 0.8629952127770177, + "dot_f1": 0.7928020565552699, + "dot_f1_threshold": 0.8031163215637207, + "dot_precision": 0.8158730158730159, + "dot_recall": 0.771, + "euclidean_accuracy": 0.9960594059405941, + "euclidean_accuracy_threshold": 0.6219213604927063, + "euclidean_ap": 0.8629950883344697, + "euclidean_f1": 0.7928020565552699, + "euclidean_f1_threshold": 0.6275087594985962, + "euclidean_precision": 0.8158730158730159, + "euclidean_recall": 0.771, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8697106165735021, + "manhattan_accuracy": 0.9961683168316832, + "manhattan_accuracy_threshold": 21.90611457824707, + "manhattan_ap": 0.8697106165735021, + "manhattan_f1": 0.7976999477260847, + "manhattan_f1_threshold": 22.14816665649414, + "manhattan_precision": 0.8357064622124863, + "manhattan_recall": 0.763, + "max_accuracy": 0.9961683168316832, + "max_ap": 0.8697106165735021, + "max_f1": 0.7976999477260847, + "max_precision": 0.8357064622124863, + "max_recall": 0.771, + "similarity_accuracy": 0.9960594059405941, + "similarity_accuracy_threshold": 0.8066068887710571, + "similarity_ap": 0.8629950883344696, + "similarity_f1": 0.7928020565552699, + "similarity_f1_threshold": 0.8031163215637207, + "similarity_precision": 0.8158730158730159, + "similarity_recall": 0.771 + } + ] + }, + "task_name": "SprintDuplicateQuestions" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/StackOverflowDupQuestions.json b/ai-sage__Giga-Embeddings-instruct/external/StackOverflowDupQuestions.json new file mode 100644 index 000000000..56f95d669 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/StackOverflowDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69", + "evaluation_time": 142.86432337760925, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47715463646129497, + "map": 0.47715463646129497, + "mrr": 0.48289287305831424, + "nAUC_map_diff1": 0.3522097553991841, + "nAUC_map_max": 0.11301549750395233, + "nAUC_map_std": 0.10780322232649885, + "nAUC_mrr_diff1": 0.3603068695413537, + "nAUC_mrr_max": 0.11937927361734656, + "nAUC_mrr_std": 0.10594734349234532 + } + ] + }, + "task_name": "StackOverflowDupQuestions" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/SummEval.json b/ai-sage__Giga-Embeddings-instruct/external/SummEval.json new file mode 100644 index 000000000..159bf1156 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/SummEval.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "evaluation_time": 29.611528873443604, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_pearson": 0.29388463097239503, + "cosine_spearman": 0.2785662716499775, + "dot_pearson": 0.29388462913794344, + "dot_spearman": 0.27888760562108095, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2785662716499775, + "pearson": 0.29388463097239503, + "spearman": 0.2785662716499775 + } + ] + }, + "task_name": "SummEval" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/TERRa.json b/ai-sage__Giga-Embeddings-instruct/external/TERRa.json new file mode 100644 index 000000000..65b3e5bc8 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 12.187905311584473, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.6091205211726385, + "cosine_accuracy_threshold": 0.7189492583274841, + "cosine_ap": 0.5997523222656755, + "cosine_f1": 0.6787330316742081, + "cosine_f1_threshold": 0.4770580530166626, + "cosine_precision": 0.5190311418685121, + "cosine_recall": 0.9803921568627451, + "dot_accuracy": 0.6091205211726385, + "dot_accuracy_threshold": 0.7189492583274841, + "dot_ap": 0.5997523222656755, + "dot_f1": 0.6787330316742081, + "dot_f1_threshold": 0.477058082818985, + "dot_precision": 0.5190311418685121, + "dot_recall": 0.9803921568627451, + "euclidean_accuracy": 0.6091205211726385, + "euclidean_accuracy_threshold": 0.7497339844703674, + "euclidean_ap": 0.5997523222656755, + "euclidean_f1": 0.6787330316742081, + "euclidean_f1_threshold": 1.0226843357086182, + "euclidean_precision": 0.5190311418685121, + "euclidean_recall": 0.9803921568627451, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6045665639652315, + "manhattan_accuracy": 0.6156351791530945, + "manhattan_accuracy_threshold": 26.853965759277344, + "manhattan_ap": 0.6045665639652315, + "manhattan_f1": 0.6772009029345373, + "manhattan_f1_threshold": 36.920753479003906, + "manhattan_precision": 0.5172413793103449, + "manhattan_recall": 0.9803921568627451, + "max_accuracy": 0.6156351791530945, + "max_ap": 0.6045665639652315, + "max_f1": 0.6787330316742081, + "max_precision": 0.5190311418685121, + "max_recall": 0.9803921568627451, + "similarity_accuracy": 0.6091205211726385, + "similarity_accuracy_threshold": 0.7189492583274841, + "similarity_ap": 0.5997523222656755, + "similarity_f1": 0.6787330316742081, + "similarity_f1_threshold": 0.4770580530166626, + "similarity_precision": 0.5190311418685121, + "similarity_recall": 0.9803921568627451 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/TRECCOVID.json b/ai-sage__Giga-Embeddings-instruct/external/TRECCOVID.json new file mode 100644 index 000000000..1342316e1 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "evaluation_time": 2109.2000889778137, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.68527, + "map_at_1": 0.00228, + "map_at_10": 0.01631, + "map_at_100": 0.07999, + "map_at_1000": 0.19209, + "map_at_20": 0.02655, + "map_at_3": 0.00562, + "map_at_5": 0.00914, + "mrr_at_1": 0.84, + "mrr_at_10": 0.9166666666666665, + "mrr_at_100": 0.9166666666666665, + "mrr_at_1000": 0.9166666666666665, + "mrr_at_20": 0.9166666666666665, + "mrr_at_3": 0.9166666666666665, + "mrr_at_5": 0.9166666666666665, + "nauc_map_at_1000_diff1": -0.249289165556578, + "nauc_map_at_1000_max": 0.3682343869931691, + "nauc_map_at_1000_std": 0.6976693302223156, + "nauc_map_at_100_diff1": -0.1493215489004332, + "nauc_map_at_100_max": 0.26903137343383193, + "nauc_map_at_100_std": 0.5193554226264329, + "nauc_map_at_10_diff1": -0.09589639096178357, + "nauc_map_at_10_max": 0.009848575880881552, + "nauc_map_at_10_std": 0.11949732717032334, + "nauc_map_at_1_diff1": -0.0002833948708031013, + "nauc_map_at_1_max": -0.09585207836823689, + "nauc_map_at_1_std": -0.0549929878351851, + "nauc_map_at_20_diff1": -0.10488150815671766, + "nauc_map_at_20_max": 0.07636306025857953, + "nauc_map_at_20_std": 0.21697516125766192, + "nauc_map_at_3_diff1": -0.12564747985952956, + "nauc_map_at_3_max": -0.05974361276237696, + "nauc_map_at_3_std": 0.02104636022300902, + "nauc_map_at_5_diff1": -0.11586078080097173, + "nauc_map_at_5_max": -0.036533992703743474, + "nauc_map_at_5_std": 0.062168626736002285, + "nauc_mrr_at_1000_diff1": -0.29749167156574197, + "nauc_mrr_at_1000_max": 0.2524789339604172, + "nauc_mrr_at_1000_std": 0.4755242014501294, + "nauc_mrr_at_100_diff1": -0.29749167156574197, + "nauc_mrr_at_100_max": 0.2524789339604172, + "nauc_mrr_at_100_std": 0.4755242014501294, + "nauc_mrr_at_10_diff1": -0.29749167156574197, + "nauc_mrr_at_10_max": 0.2524789339604172, + "nauc_mrr_at_10_std": 0.4755242014501294, + "nauc_mrr_at_1_diff1": -0.23429062346588084, + "nauc_mrr_at_1_max": 0.2392611683848804, + "nauc_mrr_at_1_std": 0.4582719685812465, + "nauc_mrr_at_20_diff1": -0.29749167156574197, + "nauc_mrr_at_20_max": 0.2524789339604172, + "nauc_mrr_at_20_std": 0.4755242014501294, + "nauc_mrr_at_3_diff1": -0.29749167156574197, + "nauc_mrr_at_3_max": 0.2524789339604172, + "nauc_mrr_at_3_std": 0.4755242014501294, + "nauc_mrr_at_5_diff1": -0.29749167156574197, + "nauc_mrr_at_5_max": 0.2524789339604172, + "nauc_mrr_at_5_std": 0.4755242014501294, + "nauc_ndcg_at_1000_diff1": -0.22983980583535282, + "nauc_ndcg_at_1000_max": 0.297425796111613, + "nauc_ndcg_at_1000_std": 0.66799739795771, + "nauc_ndcg_at_100_diff1": -0.24302637816697098, + "nauc_ndcg_at_100_max": 0.3502366417892295, + "nauc_ndcg_at_100_std": 0.7311481424467178, + "nauc_ndcg_at_10_diff1": -0.3628539128782605, + "nauc_ndcg_at_10_max": 0.22467365655768293, + "nauc_ndcg_at_10_std": 0.5701963154817496, + "nauc_ndcg_at_1_diff1": -0.3139088783455539, + "nauc_ndcg_at_1_max": 0.34128623640779926, + "nauc_ndcg_at_1_std": 0.42548409797734443, + "nauc_ndcg_at_20_diff1": -0.3072488816634815, + "nauc_ndcg_at_20_max": 0.26976879472234244, + "nauc_ndcg_at_20_std": 0.6261724664065121, + "nauc_ndcg_at_3_diff1": -0.42816038008840906, + "nauc_ndcg_at_3_max": 0.22661518863218702, + "nauc_ndcg_at_3_std": 0.4396298336797927, + "nauc_ndcg_at_5_diff1": -0.41331907905444387, + "nauc_ndcg_at_5_max": 0.21470060099934962, + "nauc_ndcg_at_5_std": 0.5268059068309917, + "nauc_precision_at_1000_diff1": -0.25175496297929095, + "nauc_precision_at_1000_max": 0.31092099302925974, + "nauc_precision_at_1000_std": 0.5528335657441642, + "nauc_precision_at_100_diff1": -0.23822350637908649, + "nauc_precision_at_100_max": 0.3882935361601115, + "nauc_precision_at_100_std": 0.7416286052994596, + "nauc_precision_at_10_diff1": -0.31256678895241896, + "nauc_precision_at_10_max": 0.1862764539335249, + "nauc_precision_at_10_std": 0.5508807879168457, + "nauc_precision_at_1_diff1": -0.23429062346588084, + "nauc_precision_at_1_max": 0.2392611683848804, + "nauc_precision_at_1_std": 0.4582719685812465, + "nauc_precision_at_20_diff1": -0.25457473488205645, + "nauc_precision_at_20_max": 0.2756443770951519, + "nauc_precision_at_20_std": 0.6213251843174319, + "nauc_precision_at_3_diff1": -0.43087270767915964, + "nauc_precision_at_3_max": 0.10025826350309319, + "nauc_precision_at_3_std": 0.3625350699541067, + "nauc_precision_at_5_diff1": -0.401801527075686, + "nauc_precision_at_5_max": 0.14947044956076014, + "nauc_precision_at_5_std": 0.5097875933897098, + "nauc_recall_at_1000_diff1": -0.21578407961752355, + "nauc_recall_at_1000_max": 0.26522130981068626, + "nauc_recall_at_1000_std": 0.5569555665357345, + "nauc_recall_at_100_diff1": -0.10077178296188566, + "nauc_recall_at_100_max": 0.17058718750934398, + "nauc_recall_at_100_std": 0.36697069293841245, + "nauc_recall_at_10_diff1": -0.0687042245147553, + "nauc_recall_at_10_max": -0.04148521545209006, + "nauc_recall_at_10_std": 0.06698236085537414, + "nauc_recall_at_1_diff1": -0.0002833948708031013, + "nauc_recall_at_1_max": -0.09585207836823689, + "nauc_recall_at_1_std": -0.0549929878351851, + "nauc_recall_at_20_diff1": -0.058202480450748974, + "nauc_recall_at_20_max": 0.028626171367487935, + "nauc_recall_at_20_std": 0.13449577793915923, + "nauc_recall_at_3_diff1": -0.09773766102342228, + "nauc_recall_at_3_max": -0.05583683668287281, + "nauc_recall_at_3_std": 0.0010625733818166493, + "nauc_recall_at_5_diff1": -0.07531560179183815, + "nauc_recall_at_5_max": -0.0510431775402095, + "nauc_recall_at_5_std": 0.029170078559993304, + "ndcg_at_1": 0.77, + "ndcg_at_10": 0.68527, + "ndcg_at_100": 0.48405, + "ndcg_at_1000": 0.44595, + "ndcg_at_20": 0.62306, + "ndcg_at_3": 0.72939, + "ndcg_at_5": 0.72416, + "precision_at_1": 0.84, + "precision_at_10": 0.73, + "precision_at_100": 0.4908, + "precision_at_1000": 0.19836, + "precision_at_20": 0.647, + "precision_at_3": 0.78, + "precision_at_5": 0.78, + "recall_at_1": 0.00228, + "recall_at_10": 0.01881, + "recall_at_100": 0.11863, + "recall_at_1000": 0.42081, + "recall_at_20": 0.03252, + "recall_at_3": 0.00595, + "recall_at_5": 0.01016 + } + ] + }, + "task_name": "TRECCOVID" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/Touche2020.json b/ai-sage__Giga-Embeddings-instruct/external/Touche2020.json new file mode 100644 index 000000000..7266c593e --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "evaluation_time": 7608.2329087257385, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.13937, + "map_at_1": 0.01293, + "map_at_10": 0.05743, + "map_at_100": 0.09267, + "map_at_1000": 0.10742, + "map_at_20": 0.07303, + "map_at_3": 0.02896, + "map_at_5": 0.03939, + "mrr_at_1": 0.14285714285714285, + "mrr_at_10": 0.2615889212827988, + "mrr_at_100": 0.2776835531782348, + "mrr_at_1000": 0.2781151081519716, + "mrr_at_20": 0.2713341930178665, + "mrr_at_3": 0.22448979591836735, + "mrr_at_5": 0.2510204081632653, + "nauc_map_at_1000_diff1": -0.06418619908635835, + "nauc_map_at_1000_max": -0.2916827935908967, + "nauc_map_at_1000_std": 0.08072487536940497, + "nauc_map_at_100_diff1": -0.04085439678921278, + "nauc_map_at_100_max": -0.300503815156051, + "nauc_map_at_100_std": 0.005940142547175356, + "nauc_map_at_10_diff1": -0.06494191455409154, + "nauc_map_at_10_max": -0.3569459573312886, + "nauc_map_at_10_std": -0.12779839188453063, + "nauc_map_at_1_diff1": -0.023057823615493914, + "nauc_map_at_1_max": -0.3366603052318725, + "nauc_map_at_1_std": -0.13676412668949378, + "nauc_map_at_20_diff1": -0.06331035661257554, + "nauc_map_at_20_max": -0.32013394348643337, + "nauc_map_at_20_std": -0.06815509360579602, + "nauc_map_at_3_diff1": -0.01133178599145889, + "nauc_map_at_3_max": -0.29760432491365213, + "nauc_map_at_3_std": -0.1996410832006419, + "nauc_map_at_5_diff1": -0.03985229627379757, + "nauc_map_at_5_max": -0.31662188888720827, + "nauc_map_at_5_std": -0.17887594061086987, + "nauc_mrr_at_1000_diff1": 0.1301193356074134, + "nauc_mrr_at_1000_max": -0.2295219247566709, + "nauc_mrr_at_1000_std": -0.04671643535232005, + "nauc_mrr_at_100_diff1": 0.13069921779672386, + "nauc_mrr_at_100_max": -0.22883092040889622, + "nauc_mrr_at_100_std": -0.04516298889631202, + "nauc_mrr_at_10_diff1": 0.11578681649426106, + "nauc_mrr_at_10_max": -0.24202645693833943, + "nauc_mrr_at_10_std": -0.052420633381824504, + "nauc_mrr_at_1_diff1": 0.13128690188364892, + "nauc_mrr_at_1_max": -0.2800109204310416, + "nauc_mrr_at_1_std": -0.05361067751504759, + "nauc_mrr_at_20_diff1": 0.12991169014806242, + "nauc_mrr_at_20_max": -0.23155024458757023, + "nauc_mrr_at_20_std": -0.04661389325134778, + "nauc_mrr_at_3_diff1": 0.1391314308295526, + "nauc_mrr_at_3_max": -0.22375537639765625, + "nauc_mrr_at_3_std": -0.14956409173121715, + "nauc_mrr_at_5_diff1": 0.12901595857173692, + "nauc_mrr_at_5_max": -0.23996408215558893, + "nauc_mrr_at_5_std": -0.0649836542830797, + "nauc_ndcg_at_1000_diff1": 0.057799908845745554, + "nauc_ndcg_at_1000_max": -0.13966564935992384, + "nauc_ndcg_at_1000_std": 0.42665694703842766, + "nauc_ndcg_at_100_diff1": 0.06234930855525906, + "nauc_ndcg_at_100_max": -0.2438418504911329, + "nauc_ndcg_at_100_std": 0.18790399332280155, + "nauc_ndcg_at_10_diff1": 0.07232052069452057, + "nauc_ndcg_at_10_max": -0.30880638325411264, + "nauc_ndcg_at_10_std": -0.0035569871365077024, + "nauc_ndcg_at_1_diff1": 0.16555725385111922, + "nauc_ndcg_at_1_max": -0.26013786248753346, + "nauc_ndcg_at_1_std": -0.03928475675217327, + "nauc_ndcg_at_20_diff1": 0.056166596541158936, + "nauc_ndcg_at_20_max": -0.30436511640651526, + "nauc_ndcg_at_20_std": -0.0019102960481939774, + "nauc_ndcg_at_3_diff1": 0.1703885024922266, + "nauc_ndcg_at_3_max": -0.19055349937590332, + "nauc_ndcg_at_3_std": -0.13228542865846385, + "nauc_ndcg_at_5_diff1": 0.14801775219525853, + "nauc_ndcg_at_5_max": -0.21249573302059271, + "nauc_ndcg_at_5_std": -0.036159018353139165, + "nauc_precision_at_1000_diff1": 0.007444573789123141, + "nauc_precision_at_1000_max": 0.3369720059833317, + "nauc_precision_at_1000_std": 0.4916550611043211, + "nauc_precision_at_100_diff1": -0.0019658777720009285, + "nauc_precision_at_100_max": -0.021519877499975908, + "nauc_precision_at_100_std": 0.559937277791322, + "nauc_precision_at_10_diff1": 0.0038911540387349783, + "nauc_precision_at_10_max": -0.2820658920522594, + "nauc_precision_at_10_std": 0.12159116774260384, + "nauc_precision_at_1_diff1": 0.13128690188364892, + "nauc_precision_at_1_max": -0.2800109204310416, + "nauc_precision_at_1_std": -0.05361067751504759, + "nauc_precision_at_20_diff1": 0.015463849480628908, + "nauc_precision_at_20_max": -0.19428037373700505, + "nauc_precision_at_20_std": 0.17118725876441906, + "nauc_precision_at_3_diff1": 0.13572269577844048, + "nauc_precision_at_3_max": -0.19254609748285048, + "nauc_precision_at_3_std": -0.16101483700594602, + "nauc_precision_at_5_diff1": 0.1044042220817277, + "nauc_precision_at_5_max": -0.2041453930267137, + "nauc_precision_at_5_std": 0.010506363193249282, + "nauc_recall_at_1000_diff1": 0.05452672989062937, + "nauc_recall_at_1000_max": 0.09433825474933832, + "nauc_recall_at_1000_std": 0.8307846603694947, + "nauc_recall_at_100_diff1": 0.025352605982490346, + "nauc_recall_at_100_max": -0.21695436099245338, + "nauc_recall_at_100_std": 0.3034608369582977, + "nauc_recall_at_10_diff1": -0.038610239417771135, + "nauc_recall_at_10_max": -0.35928264333817317, + "nauc_recall_at_10_std": -0.04431168646074323, + "nauc_recall_at_1_diff1": -0.023057823615493914, + "nauc_recall_at_1_max": -0.3366603052318725, + "nauc_recall_at_1_std": -0.13676412668949378, + "nauc_recall_at_20_diff1": -0.020569660419185166, + "nauc_recall_at_20_max": -0.3128343777931168, + "nauc_recall_at_20_std": 0.0073285989331320096, + "nauc_recall_at_3_diff1": -0.011280951425624476, + "nauc_recall_at_3_max": -0.29014124779660905, + "nauc_recall_at_3_std": -0.2440550848467014, + "nauc_recall_at_5_diff1": -0.005347797031107686, + "nauc_recall_at_5_max": -0.31129644592060185, + "nauc_recall_at_5_std": -0.14423630504744495, + "ndcg_at_1": 0.13265, + "ndcg_at_10": 0.13937, + "ndcg_at_100": 0.23186, + "ndcg_at_1000": 0.36565, + "ndcg_at_20": 0.15666, + "ndcg_at_3": 0.13664, + "ndcg_at_5": 0.13555, + "precision_at_1": 0.14286, + "precision_at_10": 0.13469, + "precision_at_100": 0.05061, + "precision_at_1000": 0.01367, + "precision_at_20": 0.10918, + "precision_at_3": 0.15646, + "precision_at_5": 0.15102, + "recall_at_1": 0.01293, + "recall_at_10": 0.11106, + "recall_at_100": 0.34054, + "recall_at_1000": 0.74773, + "recall_at_20": 0.1654, + "recall_at_3": 0.03967, + "recall_at_5": 0.06354 + } + ] + }, + "task_name": "Touche2020" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/ToxicConversationsClassification.json b/ai-sage__Giga-Embeddings-instruct/external/ToxicConversationsClassification.json new file mode 100644 index 000000000..4170655ba --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "evaluation_time": 112.77060008049011, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.753662109375, + "ap": 0.17520455604109123, + "ap_weighted": 0.17520455604109123, + "f1": 0.5870183440471106, + "f1_weighted": 0.8058084750047005, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.753662109375, + "scores_per_experiment": [ + { + "accuracy": 0.79052734375, + "ap": 0.18550259539842875, + "ap_weighted": 0.18550259539842875, + "f1": 0.6111431989494663, + "f1_weighted": 0.833471211628201 + }, + { + "accuracy": 0.79296875, + "ap": 0.17532782065372446, + "ap_weighted": 0.17532782065372446, + "f1": 0.6064044848073886, + "f1_weighted": 0.8345156856528225 + }, + { + "accuracy": 0.78759765625, + "ap": 0.19898302441259902, + "ap_weighted": 0.19898302441259902, + "f1": 0.6169271354028935, + "f1_weighted": 0.8321692553300418 + }, + { + "accuracy": 0.7900390625, + "ap": 0.18179637706388638, + "ap_weighted": 0.18179637706388638, + "f1": 0.6087901387252628, + "f1_weighted": 0.8329459095915805 + }, + { + "accuracy": 0.7177734375, + "ap": 0.1642847118263473, + "ap_weighted": 0.1642847118263473, + "f1": 0.5633197824980264, + "f1_weighted": 0.7819389132233158 + }, + { + "accuracy": 0.58544921875, + "ap": 0.13795217062561188, + "ap_weighted": 0.13795217062561188, + "f1": 0.4820640205369013, + "f1_weighted": 0.6768575524706524 + }, + { + "accuracy": 0.84716796875, + "ap": 0.20613781089523758, + "ap_weighted": 0.20613781089523758, + "f1": 0.6512532921599126, + "f1_weighted": 0.8712903621542518 + }, + { + "accuracy": 0.7080078125, + "ap": 0.16690190322769644, + "ap_weighted": 0.16690190322769644, + "f1": 0.5599275591456578, + "f1_weighted": 0.7748184318585247 + }, + { + "accuracy": 0.7099609375, + "ap": 0.15544329616845642, + "ap_weighted": 0.15544329616845642, + "f1": 0.5544245045425177, + "f1_weighted": 0.776031785880286 + }, + { + "accuracy": 0.80712890625, + "ap": 0.17971585013892388, + "ap_weighted": 0.17971585013892388, + "f1": 0.6159293237030802, + "f1_weighted": 0.8440456422573291 + } + ] + } + ] + }, + "task_name": "ToxicConversationsClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/TweetSentimentExtractionClassification.json b/ai-sage__Giga-Embeddings-instruct/external/TweetSentimentExtractionClassification.json new file mode 100644 index 000000000..6b694dcac --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "evaluation_time": 100.68703365325928, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "accuracy": 0.6764855687606113, + "f1": 0.6795235466328753, + "f1_weighted": 0.6705421511780087, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6764855687606113, + "scores_per_experiment": [ + { + "accuracy": 0.6808149405772496, + "f1": 0.6829033973065707, + "f1_weighted": 0.6729519350856437 + }, + { + "accuracy": 0.6711941143180532, + "f1": 0.6759708550260882, + "f1_weighted": 0.667247021162235 + }, + { + "accuracy": 0.6955291454442558, + "f1": 0.698728115334163, + "f1_weighted": 0.6909704990395982 + }, + { + "accuracy": 0.6870401810979061, + "f1": 0.6903661000114479, + "f1_weighted": 0.6820052442948252 + }, + { + "accuracy": 0.688737973967176, + "f1": 0.6918988805928747, + "f1_weighted": 0.6838292441377174 + }, + { + "accuracy": 0.6853423882286361, + "f1": 0.6894119907491193, + "f1_weighted": 0.6809914430736943 + }, + { + "accuracy": 0.6516694963214488, + "f1": 0.6531919196890313, + "f1_weighted": 0.6437532416625343 + }, + { + "accuracy": 0.6994906621392191, + "f1": 0.701033796926643, + "f1_weighted": 0.6927694773217433 + }, + { + "accuracy": 0.6516694963214488, + "f1": 0.6534427581747949, + "f1_weighted": 0.6414815776188321 + }, + { + "accuracy": 0.6533672891907187, + "f1": 0.6582876525180205, + "f1_weighted": 0.6494218283832637 + } + ] + } + ] + }, + "task_name": "TweetSentimentExtractionClassification" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/TwitterSemEval2015.json b/ai-sage__Giga-Embeddings-instruct/external/TwitterSemEval2015.json new file mode 100644 index 000000000..7cb3dd201 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "evaluation_time": 34.570056200027466, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_accuracy": 0.8314358943792096, + "cosine_accuracy_threshold": 0.8220133781433105, + "cosine_ap": 0.6342168849627343, + "cosine_f1": 0.6019512195121952, + "cosine_f1_threshold": 0.7819918394088745, + "cosine_precision": 0.5596371882086167, + "cosine_recall": 0.6511873350923483, + "dot_accuracy": 0.8314358943792096, + "dot_accuracy_threshold": 0.8220133781433105, + "dot_ap": 0.6342166690978694, + "dot_f1": 0.6019512195121952, + "dot_f1_threshold": 0.7819918394088745, + "dot_precision": 0.5596371882086167, + "dot_recall": 0.6511873350923483, + "euclidean_accuracy": 0.8314358943792096, + "euclidean_accuracy_threshold": 0.5966349244117737, + "euclidean_ap": 0.6342169082838296, + "euclidean_f1": 0.6019512195121952, + "euclidean_f1_threshold": 0.6603152751922607, + "euclidean_precision": 0.5596371882086167, + "euclidean_recall": 0.6511873350923483, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6356216134737833, + "manhattan_accuracy": 0.8317339214400667, + "manhattan_accuracy_threshold": 21.31719970703125, + "manhattan_ap": 0.6356216134737833, + "manhattan_f1": 0.6042120551924474, + "manhattan_f1_threshold": 23.70509910583496, + "manhattan_precision": 0.5581395348837209, + "manhattan_recall": 0.6585751978891821, + "max_accuracy": 0.8317339214400667, + "max_ap": 0.6356216134737833, + "max_f1": 0.6042120551924474, + "max_precision": 0.5596371882086167, + "max_recall": 0.6585751978891821, + "similarity_accuracy": 0.8314358943792096, + "similarity_accuracy_threshold": 0.8220133781433105, + "similarity_ap": 0.6342168849627343, + "similarity_f1": 0.6019512195121952, + "similarity_f1_threshold": 0.7819918394088745, + "similarity_precision": 0.5596371882086167, + "similarity_recall": 0.6511873350923483 + } + ] + }, + "task_name": "TwitterSemEval2015" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/TwitterURLCorpus.json b/ai-sage__Giga-Embeddings-instruct/external/TwitterURLCorpus.json new file mode 100644 index 000000000..b77b0c379 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "evaluation_time": 97.49273991584778, + "kg_co2_emissions": null, + "mteb_version": "1.15.3", + "scores": { + "test": [ + { + "cosine_accuracy": 0.8878798463150541, + "cosine_accuracy_threshold": 0.7996333837509155, + "cosine_ap": 0.8619745333095996, + "cosine_f1": 0.7774792925901053, + "cosine_f1_threshold": 0.7778993844985962, + "cosine_precision": 0.7542348342261473, + "cosine_recall": 0.8022020326455189, + "dot_accuracy": 0.8878798463150541, + "dot_accuracy_threshold": 0.7996333241462708, + "dot_ap": 0.8619745428025063, + "dot_f1": 0.7774792925901053, + "dot_f1_threshold": 0.7778993844985962, + "dot_precision": 0.7542348342261473, + "dot_recall": 0.8022020326455189, + "euclidean_accuracy": 0.8878798463150541, + "euclidean_accuracy_threshold": 0.6330349445343018, + "euclidean_ap": 0.8619745308448735, + "euclidean_f1": 0.7774792925901053, + "euclidean_f1_threshold": 0.6664841771125793, + "euclidean_precision": 0.7542348342261473, + "euclidean_recall": 0.8022020326455189, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8625813794250403, + "manhattan_accuracy": 0.8879962743043428, + "manhattan_accuracy_threshold": 22.39702606201172, + "manhattan_ap": 0.8625813794250403, + "manhattan_f1": 0.7784831272970264, + "manhattan_f1_threshold": 23.931716918945312, + "manhattan_precision": 0.7516667861495447, + "manhattan_recall": 0.8072836464428703, + "max_accuracy": 0.8879962743043428, + "max_ap": 0.8625813794250403, + "max_f1": 0.7784831272970264, + "max_precision": 0.7542348342261473, + "max_recall": 0.8072836464428703, + "similarity_accuracy": 0.8878798463150541, + "similarity_accuracy_threshold": 0.7996333837509155, + "similarity_ap": 0.8619745333095996, + "similarity_f1": 0.7774792925901053, + "similarity_f1_threshold": 0.7778993844985962, + "similarity_precision": 0.7542348342261473, + "similarity_recall": 0.8022020326455189 + } + ] + }, + "task_name": "TwitterURLCorpus" +} \ No newline at end of file diff --git a/ai-sage__Giga-Embeddings-instruct/external/model_meta.json b/ai-sage__Giga-Embeddings-instruct/external/model_meta.json new file mode 100644 index 000000000..91cdd0907 --- /dev/null +++ b/ai-sage__Giga-Embeddings-instruct/external/model_meta.json @@ -0,0 +1 @@ +{"name": "ai-sage/Giga-Embeddings-instruct", "revision": "4e5f0d19d4d2cd70558f2827193713c1862e118c", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": ["Sentence Transformers"], "loader": null}